Lines Matching +full:ctx +full:- +full:asid
1 // SPDX-License-Identifier: GPL-2.0
3 * Copyright (C) 2018-2020 Intel Corporation.
70 u64 last, u32 asid);
76 return as->id; in iotlb_to_asid()
79 static struct vhost_vdpa_as *asid_to_as(struct vhost_vdpa *v, u32 asid) in asid_to_as() argument
81 struct hlist_head *head = &v->as[asid % VHOST_VDPA_IOTLB_BUCKETS]; in asid_to_as()
85 if (as->id == asid) in asid_to_as()
91 static struct vhost_iotlb *asid_to_iotlb(struct vhost_vdpa *v, u32 asid) in asid_to_iotlb() argument
93 struct vhost_vdpa_as *as = asid_to_as(v, asid); in asid_to_iotlb()
98 return &as->iotlb; in asid_to_iotlb()
101 static struct vhost_vdpa_as *vhost_vdpa_alloc_as(struct vhost_vdpa *v, u32 asid) in vhost_vdpa_alloc_as() argument
103 struct hlist_head *head = &v->as[asid % VHOST_VDPA_IOTLB_BUCKETS]; in vhost_vdpa_alloc_as()
106 if (asid_to_as(v, asid)) in vhost_vdpa_alloc_as()
109 if (asid >= v->vdpa->nas) in vhost_vdpa_alloc_as()
116 vhost_iotlb_init(&as->iotlb, 0, 0); in vhost_vdpa_alloc_as()
117 as->id = asid; in vhost_vdpa_alloc_as()
118 hlist_add_head(&as->hash_link, head); in vhost_vdpa_alloc_as()
124 u32 asid) in vhost_vdpa_find_alloc_as() argument
126 struct vhost_vdpa_as *as = asid_to_as(v, asid); in vhost_vdpa_find_alloc_as()
131 return vhost_vdpa_alloc_as(v, asid); in vhost_vdpa_find_alloc_as()
134 static int vhost_vdpa_remove_as(struct vhost_vdpa *v, u32 asid) in vhost_vdpa_remove_as() argument
136 struct vhost_vdpa_as *as = asid_to_as(v, asid); in vhost_vdpa_remove_as()
139 return -EINVAL; in vhost_vdpa_remove_as()
141 hlist_del(&as->hash_link); in vhost_vdpa_remove_as()
142 vhost_vdpa_iotlb_unmap(v, &as->iotlb, 0ULL, 0ULL - 1, asid); in vhost_vdpa_remove_as()
152 struct vhost_vdpa *v = container_of(vq->dev, struct vhost_vdpa, vdev); in handle_vq_kick()
153 const struct vdpa_config_ops *ops = v->vdpa->config; in handle_vq_kick()
155 ops->kick_vq(v->vdpa, vq - v->vqs); in handle_vq_kick()
161 struct eventfd_ctx *call_ctx = vq->call_ctx.ctx; in vhost_vdpa_virtqueue_cb()
172 struct eventfd_ctx *config_ctx = v->config_ctx; in vhost_vdpa_config_cb()
182 struct vhost_virtqueue *vq = &v->vqs[qid]; in vhost_vdpa_setup_vq_irq()
183 const struct vdpa_config_ops *ops = v->vdpa->config; in vhost_vdpa_setup_vq_irq()
184 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_setup_vq_irq()
187 if (!ops->get_vq_irq) in vhost_vdpa_setup_vq_irq()
190 irq = ops->get_vq_irq(vdpa, qid); in vhost_vdpa_setup_vq_irq()
194 if (!vq->call_ctx.ctx) in vhost_vdpa_setup_vq_irq()
197 vq->call_ctx.producer.irq = irq; in vhost_vdpa_setup_vq_irq()
198 ret = irq_bypass_register_producer(&vq->call_ctx.producer); in vhost_vdpa_setup_vq_irq()
200 dev_info(&v->dev, "vq %u, irq bypass producer (token %p) registration fails, ret = %d\n", in vhost_vdpa_setup_vq_irq()
201 qid, vq->call_ctx.producer.token, ret); in vhost_vdpa_setup_vq_irq()
206 struct vhost_virtqueue *vq = &v->vqs[qid]; in vhost_vdpa_unsetup_vq_irq()
208 irq_bypass_unregister_producer(&vq->call_ctx.producer); in vhost_vdpa_unsetup_vq_irq()
213 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_reset()
215 v->in_batch = 0; in vhost_vdpa_reset()
222 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_bind_mm()
223 const struct vdpa_config_ops *ops = vdpa->config; in vhost_vdpa_bind_mm()
225 if (!vdpa->use_va || !ops->bind_mm) in vhost_vdpa_bind_mm()
228 return ops->bind_mm(vdpa, v->vdev.mm); in vhost_vdpa_bind_mm()
233 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_unbind_mm()
234 const struct vdpa_config_ops *ops = vdpa->config; in vhost_vdpa_unbind_mm()
236 if (!vdpa->use_va || !ops->unbind_mm) in vhost_vdpa_unbind_mm()
239 ops->unbind_mm(vdpa); in vhost_vdpa_unbind_mm()
244 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_get_device_id()
245 const struct vdpa_config_ops *ops = vdpa->config; in vhost_vdpa_get_device_id()
248 device_id = ops->get_device_id(vdpa); in vhost_vdpa_get_device_id()
251 return -EFAULT; in vhost_vdpa_get_device_id()
258 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_get_status()
259 const struct vdpa_config_ops *ops = vdpa->config; in vhost_vdpa_get_status()
262 status = ops->get_status(vdpa); in vhost_vdpa_get_status()
265 return -EFAULT; in vhost_vdpa_get_status()
272 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_set_status()
273 const struct vdpa_config_ops *ops = vdpa->config; in vhost_vdpa_set_status()
275 u32 nvqs = v->nvqs; in vhost_vdpa_set_status()
280 return -EFAULT; in vhost_vdpa_set_status()
282 status_old = ops->get_status(vdpa); in vhost_vdpa_set_status()
289 return -EINVAL; in vhost_vdpa_set_status()
312 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_config_validate()
313 size_t size = vdpa->config->get_config_size(vdpa); in vhost_vdpa_config_validate()
315 if (c->len == 0 || c->off > size) in vhost_vdpa_config_validate()
316 return -EINVAL; in vhost_vdpa_config_validate()
318 if (c->len > size - c->off) in vhost_vdpa_config_validate()
319 return -E2BIG; in vhost_vdpa_config_validate()
327 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_get_config()
333 return -EFAULT; in vhost_vdpa_get_config()
335 return -EINVAL; in vhost_vdpa_get_config()
338 return -ENOMEM; in vhost_vdpa_get_config()
342 if (copy_to_user(c->buf, buf, config.len)) { in vhost_vdpa_get_config()
344 return -EFAULT; in vhost_vdpa_get_config()
354 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_set_config()
360 return -EFAULT; in vhost_vdpa_set_config()
362 return -EINVAL; in vhost_vdpa_set_config()
364 buf = vmemdup_user(c->buf, config.len); in vhost_vdpa_set_config()
376 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_can_suspend()
377 const struct vdpa_config_ops *ops = vdpa->config; in vhost_vdpa_can_suspend()
379 return ops->suspend; in vhost_vdpa_can_suspend()
384 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_can_resume()
385 const struct vdpa_config_ops *ops = vdpa->config; in vhost_vdpa_can_resume()
387 return ops->resume; in vhost_vdpa_can_resume()
392 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_get_features()
393 const struct vdpa_config_ops *ops = vdpa->config; in vhost_vdpa_get_features()
396 features = ops->get_device_features(vdpa); in vhost_vdpa_get_features()
399 return -EFAULT; in vhost_vdpa_get_features()
406 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_get_backend_features()
407 const struct vdpa_config_ops *ops = vdpa->config; in vhost_vdpa_get_backend_features()
409 if (!ops->get_backend_features) in vhost_vdpa_get_backend_features()
412 return ops->get_backend_features(vdpa); in vhost_vdpa_get_backend_features()
417 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_set_features()
418 const struct vdpa_config_ops *ops = vdpa->config; in vhost_vdpa_set_features()
419 struct vhost_dev *d = &v->vdev; in vhost_vdpa_set_features()
428 if (ops->get_status(vdpa) & VIRTIO_CONFIG_S_FEATURES_OK) in vhost_vdpa_set_features()
429 return -EBUSY; in vhost_vdpa_set_features()
432 return -EFAULT; in vhost_vdpa_set_features()
435 return -EINVAL; in vhost_vdpa_set_features()
438 actual_features = ops->get_driver_features(vdpa); in vhost_vdpa_set_features()
439 for (i = 0; i < d->nvqs; ++i) { in vhost_vdpa_set_features()
440 struct vhost_virtqueue *vq = d->vqs[i]; in vhost_vdpa_set_features()
442 mutex_lock(&vq->mutex); in vhost_vdpa_set_features()
443 vq->acked_features = actual_features; in vhost_vdpa_set_features()
444 mutex_unlock(&vq->mutex); in vhost_vdpa_set_features()
452 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_get_vring_num()
453 const struct vdpa_config_ops *ops = vdpa->config; in vhost_vdpa_get_vring_num()
456 num = ops->get_vq_num_max(vdpa); in vhost_vdpa_get_vring_num()
459 return -EFAULT; in vhost_vdpa_get_vring_num()
466 if (v->config_ctx) { in vhost_vdpa_config_put()
467 eventfd_ctx_put(v->config_ctx); in vhost_vdpa_config_put()
468 v->config_ctx = NULL; in vhost_vdpa_config_put()
476 struct eventfd_ctx *ctx; in vhost_vdpa_set_config_call() local
481 return -EFAULT; in vhost_vdpa_set_config_call()
483 ctx = fd == VHOST_FILE_UNBIND ? NULL : eventfd_ctx_fdget(fd); in vhost_vdpa_set_config_call()
484 swap(ctx, v->config_ctx); in vhost_vdpa_set_config_call()
486 if (!IS_ERR_OR_NULL(ctx)) in vhost_vdpa_set_config_call()
487 eventfd_ctx_put(ctx); in vhost_vdpa_set_config_call()
489 if (IS_ERR(v->config_ctx)) { in vhost_vdpa_set_config_call()
490 long ret = PTR_ERR(v->config_ctx); in vhost_vdpa_set_config_call()
492 v->config_ctx = NULL; in vhost_vdpa_set_config_call()
496 v->vdpa->config->set_config_cb(v->vdpa, &cb); in vhost_vdpa_set_config_call()
504 .first = v->range.first, in vhost_vdpa_get_iova_range()
505 .last = v->range.last, in vhost_vdpa_get_iova_range()
509 return -EFAULT; in vhost_vdpa_get_iova_range()
515 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_get_config_size()
516 const struct vdpa_config_ops *ops = vdpa->config; in vhost_vdpa_get_config_size()
519 size = ops->get_config_size(vdpa); in vhost_vdpa_get_config_size()
522 return -EFAULT; in vhost_vdpa_get_config_size()
529 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_get_vqs_count()
531 if (copy_to_user(argp, &vdpa->nvqs, sizeof(vdpa->nvqs))) in vhost_vdpa_get_vqs_count()
532 return -EFAULT; in vhost_vdpa_get_vqs_count()
544 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_suspend()
545 const struct vdpa_config_ops *ops = vdpa->config; in vhost_vdpa_suspend()
547 if (!ops->suspend) in vhost_vdpa_suspend()
548 return -EOPNOTSUPP; in vhost_vdpa_suspend()
550 return ops->suspend(vdpa); in vhost_vdpa_suspend()
559 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_resume()
560 const struct vdpa_config_ops *ops = vdpa->config; in vhost_vdpa_resume()
562 if (!ops->resume) in vhost_vdpa_resume()
563 return -EOPNOTSUPP; in vhost_vdpa_resume()
565 return ops->resume(vdpa); in vhost_vdpa_resume()
571 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_vring_ioctl()
572 const struct vdpa_config_ops *ops = vdpa->config; in vhost_vdpa_vring_ioctl()
584 if (idx >= v->nvqs) in vhost_vdpa_vring_ioctl()
585 return -ENOBUFS; in vhost_vdpa_vring_ioctl()
587 idx = array_index_nospec(idx, v->nvqs); in vhost_vdpa_vring_ioctl()
588 vq = &v->vqs[idx]; in vhost_vdpa_vring_ioctl()
593 return -EFAULT; in vhost_vdpa_vring_ioctl()
594 ops->set_vq_ready(vdpa, idx, s.num); in vhost_vdpa_vring_ioctl()
597 if (!ops->get_vq_group) in vhost_vdpa_vring_ioctl()
598 return -EOPNOTSUPP; in vhost_vdpa_vring_ioctl()
600 s.num = ops->get_vq_group(vdpa, idx); in vhost_vdpa_vring_ioctl()
601 if (s.num >= vdpa->ngroups) in vhost_vdpa_vring_ioctl()
602 return -EIO; in vhost_vdpa_vring_ioctl()
604 return -EFAULT; in vhost_vdpa_vring_ioctl()
608 return -EFAULT; in vhost_vdpa_vring_ioctl()
609 if (s.num >= vdpa->nas) in vhost_vdpa_vring_ioctl()
610 return -EINVAL; in vhost_vdpa_vring_ioctl()
611 if (!ops->set_group_asid) in vhost_vdpa_vring_ioctl()
612 return -EOPNOTSUPP; in vhost_vdpa_vring_ioctl()
613 return ops->set_group_asid(vdpa, idx, s.num); in vhost_vdpa_vring_ioctl()
615 r = ops->get_vq_state(v->vdpa, idx, &vq_state); in vhost_vdpa_vring_ioctl()
620 vq->last_avail_idx = vq_state.packed.last_avail_idx | in vhost_vdpa_vring_ioctl()
622 vq->last_used_idx = vq_state.packed.last_used_idx | in vhost_vdpa_vring_ioctl()
625 vq->last_avail_idx = vq_state.split.avail_index; in vhost_vdpa_vring_ioctl()
629 if (vq->call_ctx.ctx) { in vhost_vdpa_vring_ioctl()
630 if (ops->get_status(vdpa) & in vhost_vdpa_vring_ioctl()
633 vq->call_ctx.producer.token = NULL; in vhost_vdpa_vring_ioctl()
638 r = vhost_vring_ioctl(&v->vdev, cmd, argp); in vhost_vdpa_vring_ioctl()
644 if (ops->set_vq_address(vdpa, idx, in vhost_vdpa_vring_ioctl()
645 (u64)(uintptr_t)vq->desc, in vhost_vdpa_vring_ioctl()
646 (u64)(uintptr_t)vq->avail, in vhost_vdpa_vring_ioctl()
647 (u64)(uintptr_t)vq->used)) in vhost_vdpa_vring_ioctl()
648 r = -EINVAL; in vhost_vdpa_vring_ioctl()
653 vq_state.packed.last_avail_idx = vq->last_avail_idx & 0x7fff; in vhost_vdpa_vring_ioctl()
654 vq_state.packed.last_avail_counter = !!(vq->last_avail_idx & 0x8000); in vhost_vdpa_vring_ioctl()
655 vq_state.packed.last_used_idx = vq->last_used_idx & 0x7fff; in vhost_vdpa_vring_ioctl()
656 vq_state.packed.last_used_counter = !!(vq->last_used_idx & 0x8000); in vhost_vdpa_vring_ioctl()
658 vq_state.split.avail_index = vq->last_avail_idx; in vhost_vdpa_vring_ioctl()
660 r = ops->set_vq_state(vdpa, idx, &vq_state); in vhost_vdpa_vring_ioctl()
664 if (vq->call_ctx.ctx) { in vhost_vdpa_vring_ioctl()
667 cb.trigger = vq->call_ctx.ctx; in vhost_vdpa_vring_ioctl()
668 vq->call_ctx.producer.token = vq->call_ctx.ctx; in vhost_vdpa_vring_ioctl()
669 if (ops->get_status(vdpa) & in vhost_vdpa_vring_ioctl()
677 ops->set_vq_cb(vdpa, idx, &cb); in vhost_vdpa_vring_ioctl()
681 ops->set_vq_num(vdpa, idx, vq->num); in vhost_vdpa_vring_ioctl()
691 struct vhost_vdpa *v = filep->private_data; in vhost_vdpa_unlocked_ioctl()
692 struct vhost_dev *d = &v->vdev; in vhost_vdpa_unlocked_ioctl()
700 return -EFAULT; in vhost_vdpa_unlocked_ioctl()
705 return -EOPNOTSUPP; in vhost_vdpa_unlocked_ioctl()
708 return -EOPNOTSUPP; in vhost_vdpa_unlocked_ioctl()
711 return -EOPNOTSUPP; in vhost_vdpa_unlocked_ioctl()
712 vhost_set_backend_features(&v->vdev, features); in vhost_vdpa_unlocked_ioctl()
716 mutex_lock(&d->mutex); in vhost_vdpa_unlocked_ioctl()
744 if (copy_to_user(argp, &v->vdpa->ngroups, in vhost_vdpa_unlocked_ioctl()
745 sizeof(v->vdpa->ngroups))) in vhost_vdpa_unlocked_ioctl()
746 r = -EFAULT; in vhost_vdpa_unlocked_ioctl()
749 if (copy_to_user(argp, &v->vdpa->nas, sizeof(v->vdpa->nas))) in vhost_vdpa_unlocked_ioctl()
750 r = -EFAULT; in vhost_vdpa_unlocked_ioctl()
754 r = -ENOIOCTLCMD; in vhost_vdpa_unlocked_ioctl()
767 r = -EFAULT; in vhost_vdpa_unlocked_ioctl()
785 r = vhost_dev_ioctl(&v->vdev, cmd, argp); in vhost_vdpa_unlocked_ioctl()
786 if (r == -ENOIOCTLCMD) in vhost_vdpa_unlocked_ioctl()
802 mutex_unlock(&d->mutex); in vhost_vdpa_unlocked_ioctl()
806 struct vhost_iotlb_map *map, u32 asid) in vhost_vdpa_general_unmap() argument
808 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_general_unmap()
809 const struct vdpa_config_ops *ops = vdpa->config; in vhost_vdpa_general_unmap()
810 if (ops->dma_map) { in vhost_vdpa_general_unmap()
811 ops->dma_unmap(vdpa, asid, map->start, map->size); in vhost_vdpa_general_unmap()
812 } else if (ops->set_map == NULL) { in vhost_vdpa_general_unmap()
813 iommu_unmap(v->domain, map->start, map->size); in vhost_vdpa_general_unmap()
818 u64 start, u64 last, u32 asid) in vhost_vdpa_pa_unmap() argument
820 struct vhost_dev *dev = &v->vdev; in vhost_vdpa_pa_unmap()
826 pinned = PFN_DOWN(map->size); in vhost_vdpa_pa_unmap()
827 for (pfn = PFN_DOWN(map->addr); in vhost_vdpa_pa_unmap()
828 pinned > 0; pfn++, pinned--) { in vhost_vdpa_pa_unmap()
830 if (map->perm & VHOST_ACCESS_WO) in vhost_vdpa_pa_unmap()
834 atomic64_sub(PFN_DOWN(map->size), &dev->mm->pinned_vm); in vhost_vdpa_pa_unmap()
835 vhost_vdpa_general_unmap(v, map, asid); in vhost_vdpa_pa_unmap()
841 u64 start, u64 last, u32 asid) in vhost_vdpa_va_unmap() argument
847 map_file = (struct vdpa_map_file *)map->opaque; in vhost_vdpa_va_unmap()
848 fput(map_file->file); in vhost_vdpa_va_unmap()
850 vhost_vdpa_general_unmap(v, map, asid); in vhost_vdpa_va_unmap()
857 u64 last, u32 asid) in vhost_vdpa_iotlb_unmap() argument
859 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_iotlb_unmap()
861 if (vdpa->use_va) in vhost_vdpa_iotlb_unmap()
862 return vhost_vdpa_va_unmap(v, iotlb, start, last, asid); in vhost_vdpa_iotlb_unmap()
864 return vhost_vdpa_pa_unmap(v, iotlb, start, last, asid); in vhost_vdpa_iotlb_unmap()
892 struct vhost_dev *dev = &v->vdev; in vhost_vdpa_map()
893 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_map()
894 const struct vdpa_config_ops *ops = vdpa->config; in vhost_vdpa_map()
895 u32 asid = iotlb_to_asid(iotlb); in vhost_vdpa_map() local
898 r = vhost_iotlb_add_range_ctx(iotlb, iova, iova + size - 1, in vhost_vdpa_map()
903 if (ops->dma_map) { in vhost_vdpa_map()
904 r = ops->dma_map(vdpa, asid, iova, size, pa, perm, opaque); in vhost_vdpa_map()
905 } else if (ops->set_map) { in vhost_vdpa_map()
906 if (!v->in_batch) in vhost_vdpa_map()
907 r = ops->set_map(vdpa, asid, iotlb); in vhost_vdpa_map()
909 r = iommu_map(v->domain, iova, pa, size, in vhost_vdpa_map()
913 vhost_iotlb_del_range(iotlb, iova, iova + size - 1); in vhost_vdpa_map()
917 if (!vdpa->use_va) in vhost_vdpa_map()
918 atomic64_add(PFN_DOWN(size), &dev->mm->pinned_vm); in vhost_vdpa_map()
927 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_unmap()
928 const struct vdpa_config_ops *ops = vdpa->config; in vhost_vdpa_unmap()
929 u32 asid = iotlb_to_asid(iotlb); in vhost_vdpa_unmap() local
931 vhost_vdpa_iotlb_unmap(v, iotlb, iova, iova + size - 1, asid); in vhost_vdpa_unmap()
933 if (ops->set_map) { in vhost_vdpa_unmap()
934 if (!v->in_batch) in vhost_vdpa_unmap()
935 ops->set_map(vdpa, asid, iotlb); in vhost_vdpa_unmap()
944 struct vhost_dev *dev = &v->vdev; in vhost_vdpa_va_map()
950 mmap_read_lock(dev->mm); in vhost_vdpa_va_map()
953 vma = find_vma(dev->mm, uaddr); in vhost_vdpa_va_map()
955 ret = -EINVAL; in vhost_vdpa_va_map()
958 map_size = min(size, vma->vm_end - uaddr); in vhost_vdpa_va_map()
959 if (!(vma->vm_file && (vma->vm_flags & VM_SHARED) && in vhost_vdpa_va_map()
960 !(vma->vm_flags & (VM_IO | VM_PFNMAP)))) in vhost_vdpa_va_map()
965 ret = -ENOMEM; in vhost_vdpa_va_map()
968 offset = (vma->vm_pgoff << PAGE_SHIFT) + uaddr - vma->vm_start; in vhost_vdpa_va_map()
969 map_file->offset = offset; in vhost_vdpa_va_map()
970 map_file->file = get_file(vma->vm_file); in vhost_vdpa_va_map()
974 fput(map_file->file); in vhost_vdpa_va_map()
979 size -= map_size; in vhost_vdpa_va_map()
984 vhost_vdpa_unmap(v, iotlb, iova, map_iova - iova); in vhost_vdpa_va_map()
986 mmap_read_unlock(dev->mm); in vhost_vdpa_va_map()
995 struct vhost_dev *dev = &v->vdev; in vhost_vdpa_pa_map()
1008 return -ENOMEM; in vhost_vdpa_pa_map()
1015 ret = -EINVAL; in vhost_vdpa_pa_map()
1019 mmap_read_lock(dev->mm); in vhost_vdpa_pa_map()
1022 if (npages + atomic64_read(&dev->mm->pinned_vm) > lock_limit) { in vhost_vdpa_pa_map()
1023 ret = -ENOMEM; in vhost_vdpa_pa_map()
1040 ret = -ENOMEM; in vhost_vdpa_pa_map()
1055 csize = PFN_PHYS(last_pfn - map_pfn + 1); in vhost_vdpa_pa_map()
1069 pinned - i); in vhost_vdpa_pa_map()
1082 npages -= pinned; in vhost_vdpa_pa_map()
1086 ret = vhost_vdpa_map(v, iotlb, iova, PFN_PHYS(last_pfn - map_pfn + 1), in vhost_vdpa_pa_map()
1109 mmap_read_unlock(dev->mm); in vhost_vdpa_pa_map()
1120 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_process_iotlb_update()
1122 if (msg->iova < v->range.first || !msg->size || in vhost_vdpa_process_iotlb_update()
1123 msg->iova > U64_MAX - msg->size + 1 || in vhost_vdpa_process_iotlb_update()
1124 msg->iova + msg->size - 1 > v->range.last) in vhost_vdpa_process_iotlb_update()
1125 return -EINVAL; in vhost_vdpa_process_iotlb_update()
1127 if (vhost_iotlb_itree_first(iotlb, msg->iova, in vhost_vdpa_process_iotlb_update()
1128 msg->iova + msg->size - 1)) in vhost_vdpa_process_iotlb_update()
1129 return -EEXIST; in vhost_vdpa_process_iotlb_update()
1131 if (vdpa->use_va) in vhost_vdpa_process_iotlb_update()
1132 return vhost_vdpa_va_map(v, iotlb, msg->iova, msg->size, in vhost_vdpa_process_iotlb_update()
1133 msg->uaddr, msg->perm); in vhost_vdpa_process_iotlb_update()
1135 return vhost_vdpa_pa_map(v, iotlb, msg->iova, msg->size, msg->uaddr, in vhost_vdpa_process_iotlb_update()
1136 msg->perm); in vhost_vdpa_process_iotlb_update()
1139 static int vhost_vdpa_process_iotlb_msg(struct vhost_dev *dev, u32 asid, in vhost_vdpa_process_iotlb_msg() argument
1143 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_process_iotlb_msg()
1144 const struct vdpa_config_ops *ops = vdpa->config; in vhost_vdpa_process_iotlb_msg()
1149 mutex_lock(&dev->mutex); in vhost_vdpa_process_iotlb_msg()
1155 if (msg->type == VHOST_IOTLB_UPDATE || in vhost_vdpa_process_iotlb_msg()
1156 msg->type == VHOST_IOTLB_BATCH_BEGIN) { in vhost_vdpa_process_iotlb_msg()
1157 as = vhost_vdpa_find_alloc_as(v, asid); in vhost_vdpa_process_iotlb_msg()
1159 dev_err(&v->dev, "can't find and alloc asid %d\n", in vhost_vdpa_process_iotlb_msg()
1160 asid); in vhost_vdpa_process_iotlb_msg()
1161 r = -EINVAL; in vhost_vdpa_process_iotlb_msg()
1164 iotlb = &as->iotlb; in vhost_vdpa_process_iotlb_msg()
1166 iotlb = asid_to_iotlb(v, asid); in vhost_vdpa_process_iotlb_msg()
1168 if ((v->in_batch && v->batch_asid != asid) || !iotlb) { in vhost_vdpa_process_iotlb_msg()
1169 if (v->in_batch && v->batch_asid != asid) { in vhost_vdpa_process_iotlb_msg()
1170 dev_info(&v->dev, "batch id %d asid %d\n", in vhost_vdpa_process_iotlb_msg()
1171 v->batch_asid, asid); in vhost_vdpa_process_iotlb_msg()
1174 dev_err(&v->dev, "no iotlb for asid %d\n", asid); in vhost_vdpa_process_iotlb_msg()
1175 r = -EINVAL; in vhost_vdpa_process_iotlb_msg()
1179 switch (msg->type) { in vhost_vdpa_process_iotlb_msg()
1184 vhost_vdpa_unmap(v, iotlb, msg->iova, msg->size); in vhost_vdpa_process_iotlb_msg()
1187 v->batch_asid = asid; in vhost_vdpa_process_iotlb_msg()
1188 v->in_batch = true; in vhost_vdpa_process_iotlb_msg()
1191 if (v->in_batch && ops->set_map) in vhost_vdpa_process_iotlb_msg()
1192 ops->set_map(vdpa, asid, iotlb); in vhost_vdpa_process_iotlb_msg()
1193 v->in_batch = false; in vhost_vdpa_process_iotlb_msg()
1196 r = -EINVAL; in vhost_vdpa_process_iotlb_msg()
1200 mutex_unlock(&dev->mutex); in vhost_vdpa_process_iotlb_msg()
1208 struct file *file = iocb->ki_filp; in vhost_vdpa_chr_write_iter()
1209 struct vhost_vdpa *v = file->private_data; in vhost_vdpa_chr_write_iter()
1210 struct vhost_dev *dev = &v->vdev; in vhost_vdpa_chr_write_iter()
1217 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_alloc_domain()
1218 const struct vdpa_config_ops *ops = vdpa->config; in vhost_vdpa_alloc_domain()
1224 if (ops->set_map || ops->dma_map) in vhost_vdpa_alloc_domain()
1227 bus = dma_dev->bus; in vhost_vdpa_alloc_domain()
1229 return -EFAULT; in vhost_vdpa_alloc_domain()
1232 dev_warn_once(&v->dev, in vhost_vdpa_alloc_domain()
1234 return -ENOTSUPP; in vhost_vdpa_alloc_domain()
1237 v->domain = iommu_domain_alloc(bus); in vhost_vdpa_alloc_domain()
1238 if (!v->domain) in vhost_vdpa_alloc_domain()
1239 return -EIO; in vhost_vdpa_alloc_domain()
1241 ret = iommu_attach_device(v->domain, dma_dev); in vhost_vdpa_alloc_domain()
1248 iommu_domain_free(v->domain); in vhost_vdpa_alloc_domain()
1249 v->domain = NULL; in vhost_vdpa_alloc_domain()
1255 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_free_domain()
1258 if (v->domain) { in vhost_vdpa_free_domain()
1259 iommu_detach_device(v->domain, dma_dev); in vhost_vdpa_free_domain()
1260 iommu_domain_free(v->domain); in vhost_vdpa_free_domain()
1263 v->domain = NULL; in vhost_vdpa_free_domain()
1268 struct vdpa_iova_range *range = &v->range; in vhost_vdpa_set_iova_range()
1269 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_set_iova_range()
1270 const struct vdpa_config_ops *ops = vdpa->config; in vhost_vdpa_set_iova_range()
1272 if (ops->get_iova_range) { in vhost_vdpa_set_iova_range()
1273 *range = ops->get_iova_range(vdpa); in vhost_vdpa_set_iova_range()
1274 } else if (v->domain && v->domain->geometry.force_aperture) { in vhost_vdpa_set_iova_range()
1275 range->first = v->domain->geometry.aperture_start; in vhost_vdpa_set_iova_range()
1276 range->last = v->domain->geometry.aperture_end; in vhost_vdpa_set_iova_range()
1278 range->first = 0; in vhost_vdpa_set_iova_range()
1279 range->last = ULLONG_MAX; in vhost_vdpa_set_iova_range()
1286 u32 asid; in vhost_vdpa_cleanup() local
1288 for (asid = 0; asid < v->vdpa->nas; asid++) { in vhost_vdpa_cleanup()
1289 as = asid_to_as(v, asid); in vhost_vdpa_cleanup()
1291 vhost_vdpa_remove_as(v, asid); in vhost_vdpa_cleanup()
1295 vhost_dev_cleanup(&v->vdev); in vhost_vdpa_cleanup()
1296 kfree(v->vdev.vqs); in vhost_vdpa_cleanup()
1307 v = container_of(inode->i_cdev, struct vhost_vdpa, cdev); in vhost_vdpa_open()
1309 opened = atomic_cmpxchg(&v->opened, 0, 1); in vhost_vdpa_open()
1311 return -EBUSY; in vhost_vdpa_open()
1313 nvqs = v->nvqs; in vhost_vdpa_open()
1320 r = -ENOMEM; in vhost_vdpa_open()
1324 dev = &v->vdev; in vhost_vdpa_open()
1326 vqs[i] = &v->vqs[i]; in vhost_vdpa_open()
1327 vqs[i]->handle_kick = handle_vq_kick; in vhost_vdpa_open()
1328 vqs[i]->call_ctx.ctx = NULL; in vhost_vdpa_open()
1339 filep->private_data = v; in vhost_vdpa_open()
1346 atomic_dec(&v->opened); in vhost_vdpa_open()
1354 for (i = 0; i < v->nvqs; i++) in vhost_vdpa_clean_irq()
1360 struct vhost_vdpa *v = filep->private_data; in vhost_vdpa_release()
1361 struct vhost_dev *d = &v->vdev; in vhost_vdpa_release()
1363 mutex_lock(&d->mutex); in vhost_vdpa_release()
1364 filep->private_data = NULL; in vhost_vdpa_release()
1367 vhost_dev_stop(&v->vdev); in vhost_vdpa_release()
1371 mutex_unlock(&d->mutex); in vhost_vdpa_release()
1373 atomic_dec(&v->opened); in vhost_vdpa_release()
1374 complete(&v->completion); in vhost_vdpa_release()
1382 struct vhost_vdpa *v = vmf->vma->vm_file->private_data; in vhost_vdpa_fault()
1383 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_fault()
1384 const struct vdpa_config_ops *ops = vdpa->config; in vhost_vdpa_fault()
1386 struct vm_area_struct *vma = vmf->vma; in vhost_vdpa_fault()
1387 u16 index = vma->vm_pgoff; in vhost_vdpa_fault()
1389 notify = ops->get_vq_notification(vdpa, index); in vhost_vdpa_fault()
1391 return vmf_insert_pfn(vma, vmf->address & PAGE_MASK, PFN_DOWN(notify.addr)); in vhost_vdpa_fault()
1400 struct vhost_vdpa *v = vma->vm_file->private_data; in vhost_vdpa_mmap()
1401 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_mmap()
1402 const struct vdpa_config_ops *ops = vdpa->config; in vhost_vdpa_mmap()
1404 unsigned long index = vma->vm_pgoff; in vhost_vdpa_mmap()
1406 if (vma->vm_end - vma->vm_start != PAGE_SIZE) in vhost_vdpa_mmap()
1407 return -EINVAL; in vhost_vdpa_mmap()
1408 if ((vma->vm_flags & VM_SHARED) == 0) in vhost_vdpa_mmap()
1409 return -EINVAL; in vhost_vdpa_mmap()
1410 if (vma->vm_flags & VM_READ) in vhost_vdpa_mmap()
1411 return -EINVAL; in vhost_vdpa_mmap()
1413 return -EINVAL; in vhost_vdpa_mmap()
1414 if (!ops->get_vq_notification) in vhost_vdpa_mmap()
1415 return -ENOTSUPP; in vhost_vdpa_mmap()
1421 notify = ops->get_vq_notification(vdpa, index); in vhost_vdpa_mmap()
1422 if (notify.addr & (PAGE_SIZE - 1)) in vhost_vdpa_mmap()
1423 return -EINVAL; in vhost_vdpa_mmap()
1424 if (vma->vm_end - vma->vm_start != notify.size) in vhost_vdpa_mmap()
1425 return -ENOTSUPP; in vhost_vdpa_mmap()
1428 vma->vm_ops = &vhost_vdpa_vm_ops; in vhost_vdpa_mmap()
1450 ida_simple_remove(&vhost_vdpa_ida, v->minor); in vhost_vdpa_release_dev()
1451 kfree(v->vqs); in vhost_vdpa_release_dev()
1457 const struct vdpa_config_ops *ops = vdpa->config; in vhost_vdpa_probe()
1465 if (!ops->set_map && !ops->dma_map && in vhost_vdpa_probe()
1466 (vdpa->ngroups > 1 || vdpa->nas > 1)) in vhost_vdpa_probe()
1467 return -EOPNOTSUPP; in vhost_vdpa_probe()
1471 return -ENOMEM; in vhost_vdpa_probe()
1480 atomic_set(&v->opened, 0); in vhost_vdpa_probe()
1481 v->minor = minor; in vhost_vdpa_probe()
1482 v->vdpa = vdpa; in vhost_vdpa_probe()
1483 v->nvqs = vdpa->nvqs; in vhost_vdpa_probe()
1484 v->virtio_id = ops->get_device_id(vdpa); in vhost_vdpa_probe()
1486 device_initialize(&v->dev); in vhost_vdpa_probe()
1487 v->dev.release = vhost_vdpa_release_dev; in vhost_vdpa_probe()
1488 v->dev.parent = &vdpa->dev; in vhost_vdpa_probe()
1489 v->dev.devt = MKDEV(MAJOR(vhost_vdpa_major), minor); in vhost_vdpa_probe()
1490 v->vqs = kmalloc_array(v->nvqs, sizeof(struct vhost_virtqueue), in vhost_vdpa_probe()
1492 if (!v->vqs) { in vhost_vdpa_probe()
1493 r = -ENOMEM; in vhost_vdpa_probe()
1497 r = dev_set_name(&v->dev, "vhost-vdpa-%u", minor); in vhost_vdpa_probe()
1501 cdev_init(&v->cdev, &vhost_vdpa_fops); in vhost_vdpa_probe()
1502 v->cdev.owner = THIS_MODULE; in vhost_vdpa_probe()
1504 r = cdev_device_add(&v->cdev, &v->dev); in vhost_vdpa_probe()
1508 init_completion(&v->completion); in vhost_vdpa_probe()
1512 INIT_HLIST_HEAD(&v->as[i]); in vhost_vdpa_probe()
1517 put_device(&v->dev); in vhost_vdpa_probe()
1526 cdev_device_del(&v->cdev, &v->dev); in vhost_vdpa_remove()
1529 opened = atomic_cmpxchg(&v->opened, 0, 1); in vhost_vdpa_remove()
1532 wait_for_completion(&v->completion); in vhost_vdpa_remove()
1535 put_device(&v->dev); in vhost_vdpa_remove()
1551 "vhost-vdpa"); in vhost_vdpa_init()
1578 MODULE_DESCRIPTION("vDPA-based vhost backend for virtio");