1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (C) 2024 Intel Corporation
3 */
4 #define pr_fmt(fmt) "iommufd: " fmt
5
6 #include <linux/anon_inodes.h>
7 #include <linux/file.h>
8 #include <linux/fs.h>
9 #include <linux/iommufd.h>
10 #include <linux/module.h>
11 #include <linux/mutex.h>
12 #include <linux/pci.h>
13 #include <linux/poll.h>
14 #include <uapi/linux/iommufd.h>
15
16 #include "../iommu-priv.h"
17 #include "iommufd_private.h"
18
iommufd_fault_iopf_enable(struct iommufd_device * idev)19 int iommufd_fault_iopf_enable(struct iommufd_device *idev)
20 {
21 struct device *dev = idev->dev;
22 int ret;
23
24 /*
25 * Once we turn on PCI/PRI support for VF, the response failure code
26 * should not be forwarded to the hardware due to PRI being a shared
27 * resource between PF and VFs. There is no coordination for this
28 * shared capability. This waits for a vPRI reset to recover.
29 */
30 if (dev_is_pci(dev) && to_pci_dev(dev)->is_virtfn)
31 return -EINVAL;
32
33 mutex_lock(&idev->iopf_lock);
34 /* Device iopf has already been on. */
35 if (++idev->iopf_enabled > 1) {
36 mutex_unlock(&idev->iopf_lock);
37 return 0;
38 }
39
40 ret = iommu_dev_enable_feature(dev, IOMMU_DEV_FEAT_IOPF);
41 if (ret)
42 --idev->iopf_enabled;
43 mutex_unlock(&idev->iopf_lock);
44
45 return ret;
46 }
47
iommufd_fault_iopf_disable(struct iommufd_device * idev)48 void iommufd_fault_iopf_disable(struct iommufd_device *idev)
49 {
50 mutex_lock(&idev->iopf_lock);
51 if (!WARN_ON(idev->iopf_enabled == 0)) {
52 if (--idev->iopf_enabled == 0)
53 iommu_dev_disable_feature(idev->dev, IOMMU_DEV_FEAT_IOPF);
54 }
55 mutex_unlock(&idev->iopf_lock);
56 }
57
__fault_domain_attach_dev(struct iommufd_hw_pagetable * hwpt,struct iommufd_device * idev)58 static int __fault_domain_attach_dev(struct iommufd_hw_pagetable *hwpt,
59 struct iommufd_device *idev)
60 {
61 struct iommufd_attach_handle *handle;
62 int ret;
63
64 handle = kzalloc(sizeof(*handle), GFP_KERNEL);
65 if (!handle)
66 return -ENOMEM;
67
68 handle->idev = idev;
69 ret = iommu_attach_group_handle(hwpt->domain, idev->igroup->group,
70 &handle->handle);
71 if (ret)
72 kfree(handle);
73
74 return ret;
75 }
76
iommufd_fault_domain_attach_dev(struct iommufd_hw_pagetable * hwpt,struct iommufd_device * idev)77 int iommufd_fault_domain_attach_dev(struct iommufd_hw_pagetable *hwpt,
78 struct iommufd_device *idev)
79 {
80 int ret;
81
82 if (!hwpt->fault)
83 return -EINVAL;
84
85 ret = iommufd_fault_iopf_enable(idev);
86 if (ret)
87 return ret;
88
89 ret = __fault_domain_attach_dev(hwpt, idev);
90 if (ret)
91 iommufd_fault_iopf_disable(idev);
92
93 return ret;
94 }
95
iommufd_auto_response_faults(struct iommufd_hw_pagetable * hwpt,struct iommufd_attach_handle * handle)96 void iommufd_auto_response_faults(struct iommufd_hw_pagetable *hwpt,
97 struct iommufd_attach_handle *handle)
98 {
99 struct iommufd_fault *fault = hwpt->fault;
100 struct iopf_group *group, *next;
101 struct list_head free_list;
102 unsigned long index;
103
104 if (!fault)
105 return;
106 INIT_LIST_HEAD(&free_list);
107
108 mutex_lock(&fault->mutex);
109 spin_lock(&fault->lock);
110 list_for_each_entry_safe(group, next, &fault->deliver, node) {
111 if (group->attach_handle != &handle->handle)
112 continue;
113 list_move(&group->node, &free_list);
114 }
115 spin_unlock(&fault->lock);
116
117 list_for_each_entry_safe(group, next, &free_list, node) {
118 list_del(&group->node);
119 iopf_group_response(group, IOMMU_PAGE_RESP_INVALID);
120 iopf_free_group(group);
121 }
122
123 xa_for_each(&fault->response, index, group) {
124 if (group->attach_handle != &handle->handle)
125 continue;
126 xa_erase(&fault->response, index);
127 iopf_group_response(group, IOMMU_PAGE_RESP_INVALID);
128 iopf_free_group(group);
129 }
130 mutex_unlock(&fault->mutex);
131 }
132
133 static struct iommufd_attach_handle *
iommufd_device_get_attach_handle(struct iommufd_device * idev)134 iommufd_device_get_attach_handle(struct iommufd_device *idev)
135 {
136 struct iommu_attach_handle *handle;
137
138 handle = iommu_attach_handle_get(idev->igroup->group, IOMMU_NO_PASID, 0);
139 if (IS_ERR(handle))
140 return NULL;
141
142 return to_iommufd_handle(handle);
143 }
144
iommufd_fault_domain_detach_dev(struct iommufd_hw_pagetable * hwpt,struct iommufd_device * idev)145 void iommufd_fault_domain_detach_dev(struct iommufd_hw_pagetable *hwpt,
146 struct iommufd_device *idev)
147 {
148 struct iommufd_attach_handle *handle;
149
150 handle = iommufd_device_get_attach_handle(idev);
151 iommu_detach_group_handle(hwpt->domain, idev->igroup->group);
152 iommufd_auto_response_faults(hwpt, handle);
153 iommufd_fault_iopf_disable(idev);
154 kfree(handle);
155 }
156
__fault_domain_replace_dev(struct iommufd_device * idev,struct iommufd_hw_pagetable * hwpt,struct iommufd_hw_pagetable * old)157 static int __fault_domain_replace_dev(struct iommufd_device *idev,
158 struct iommufd_hw_pagetable *hwpt,
159 struct iommufd_hw_pagetable *old)
160 {
161 struct iommufd_attach_handle *handle, *curr = NULL;
162 int ret;
163
164 if (old->fault)
165 curr = iommufd_device_get_attach_handle(idev);
166
167 if (hwpt->fault) {
168 handle = kzalloc(sizeof(*handle), GFP_KERNEL);
169 if (!handle)
170 return -ENOMEM;
171
172 handle->idev = idev;
173 ret = iommu_replace_group_handle(idev->igroup->group,
174 hwpt->domain, &handle->handle);
175 } else {
176 ret = iommu_replace_group_handle(idev->igroup->group,
177 hwpt->domain, NULL);
178 }
179
180 if (!ret && curr) {
181 iommufd_auto_response_faults(old, curr);
182 kfree(curr);
183 }
184
185 return ret;
186 }
187
iommufd_fault_domain_replace_dev(struct iommufd_device * idev,struct iommufd_hw_pagetable * hwpt,struct iommufd_hw_pagetable * old)188 int iommufd_fault_domain_replace_dev(struct iommufd_device *idev,
189 struct iommufd_hw_pagetable *hwpt,
190 struct iommufd_hw_pagetable *old)
191 {
192 bool iopf_off = !hwpt->fault && old->fault;
193 bool iopf_on = hwpt->fault && !old->fault;
194 int ret;
195
196 if (iopf_on) {
197 ret = iommufd_fault_iopf_enable(idev);
198 if (ret)
199 return ret;
200 }
201
202 ret = __fault_domain_replace_dev(idev, hwpt, old);
203 if (ret) {
204 if (iopf_on)
205 iommufd_fault_iopf_disable(idev);
206 return ret;
207 }
208
209 if (iopf_off)
210 iommufd_fault_iopf_disable(idev);
211
212 return 0;
213 }
214
iommufd_fault_destroy(struct iommufd_object * obj)215 void iommufd_fault_destroy(struct iommufd_object *obj)
216 {
217 struct iommufd_fault *fault = container_of(obj, struct iommufd_fault, obj);
218 struct iopf_group *group, *next;
219 unsigned long index;
220
221 /*
222 * The iommufd object's reference count is zero at this point.
223 * We can be confident that no other threads are currently
224 * accessing this pointer. Therefore, acquiring the mutex here
225 * is unnecessary.
226 */
227 list_for_each_entry_safe(group, next, &fault->deliver, node) {
228 list_del(&group->node);
229 iopf_group_response(group, IOMMU_PAGE_RESP_INVALID);
230 iopf_free_group(group);
231 }
232 xa_for_each(&fault->response, index, group) {
233 xa_erase(&fault->response, index);
234 iopf_group_response(group, IOMMU_PAGE_RESP_INVALID);
235 iopf_free_group(group);
236 }
237 xa_destroy(&fault->response);
238 mutex_destroy(&fault->mutex);
239 }
240
iommufd_compose_fault_message(struct iommu_fault * fault,struct iommu_hwpt_pgfault * hwpt_fault,struct iommufd_device * idev,u32 cookie)241 static void iommufd_compose_fault_message(struct iommu_fault *fault,
242 struct iommu_hwpt_pgfault *hwpt_fault,
243 struct iommufd_device *idev,
244 u32 cookie)
245 {
246 hwpt_fault->flags = fault->prm.flags;
247 hwpt_fault->dev_id = idev->obj.id;
248 hwpt_fault->pasid = fault->prm.pasid;
249 hwpt_fault->grpid = fault->prm.grpid;
250 hwpt_fault->perm = fault->prm.perm;
251 hwpt_fault->addr = fault->prm.addr;
252 hwpt_fault->length = 0;
253 hwpt_fault->cookie = cookie;
254 }
255
iommufd_fault_fops_read(struct file * filep,char __user * buf,size_t count,loff_t * ppos)256 static ssize_t iommufd_fault_fops_read(struct file *filep, char __user *buf,
257 size_t count, loff_t *ppos)
258 {
259 size_t fault_size = sizeof(struct iommu_hwpt_pgfault);
260 struct iommufd_fault *fault = filep->private_data;
261 struct iommu_hwpt_pgfault data = {};
262 struct iommufd_device *idev;
263 struct iopf_group *group;
264 struct iopf_fault *iopf;
265 size_t done = 0;
266 int rc = 0;
267
268 if (*ppos || count % fault_size)
269 return -ESPIPE;
270
271 mutex_lock(&fault->mutex);
272 while ((group = iommufd_fault_deliver_fetch(fault))) {
273 if (done >= count ||
274 group->fault_count * fault_size > count - done) {
275 iommufd_fault_deliver_restore(fault, group);
276 break;
277 }
278
279 rc = xa_alloc(&fault->response, &group->cookie, group,
280 xa_limit_32b, GFP_KERNEL);
281 if (rc) {
282 iommufd_fault_deliver_restore(fault, group);
283 break;
284 }
285
286 idev = to_iommufd_handle(group->attach_handle)->idev;
287 list_for_each_entry(iopf, &group->faults, list) {
288 iommufd_compose_fault_message(&iopf->fault,
289 &data, idev,
290 group->cookie);
291 if (copy_to_user(buf + done, &data, fault_size)) {
292 xa_erase(&fault->response, group->cookie);
293 iommufd_fault_deliver_restore(fault, group);
294 rc = -EFAULT;
295 break;
296 }
297 done += fault_size;
298 }
299 }
300 mutex_unlock(&fault->mutex);
301
302 return done == 0 ? rc : done;
303 }
304
iommufd_fault_fops_write(struct file * filep,const char __user * buf,size_t count,loff_t * ppos)305 static ssize_t iommufd_fault_fops_write(struct file *filep, const char __user *buf,
306 size_t count, loff_t *ppos)
307 {
308 size_t response_size = sizeof(struct iommu_hwpt_page_response);
309 struct iommufd_fault *fault = filep->private_data;
310 struct iommu_hwpt_page_response response;
311 struct iopf_group *group;
312 size_t done = 0;
313 int rc = 0;
314
315 if (*ppos || count % response_size)
316 return -ESPIPE;
317
318 mutex_lock(&fault->mutex);
319 while (count > done) {
320 rc = copy_from_user(&response, buf + done, response_size);
321 if (rc)
322 break;
323
324 static_assert((int)IOMMUFD_PAGE_RESP_SUCCESS ==
325 (int)IOMMU_PAGE_RESP_SUCCESS);
326 static_assert((int)IOMMUFD_PAGE_RESP_INVALID ==
327 (int)IOMMU_PAGE_RESP_INVALID);
328 if (response.code != IOMMUFD_PAGE_RESP_SUCCESS &&
329 response.code != IOMMUFD_PAGE_RESP_INVALID) {
330 rc = -EINVAL;
331 break;
332 }
333
334 group = xa_erase(&fault->response, response.cookie);
335 if (!group) {
336 rc = -EINVAL;
337 break;
338 }
339
340 iopf_group_response(group, response.code);
341 iopf_free_group(group);
342 done += response_size;
343 }
344 mutex_unlock(&fault->mutex);
345
346 return done == 0 ? rc : done;
347 }
348
iommufd_fault_fops_poll(struct file * filep,struct poll_table_struct * wait)349 static __poll_t iommufd_fault_fops_poll(struct file *filep,
350 struct poll_table_struct *wait)
351 {
352 struct iommufd_fault *fault = filep->private_data;
353 __poll_t pollflags = EPOLLOUT;
354
355 poll_wait(filep, &fault->wait_queue, wait);
356 spin_lock(&fault->lock);
357 if (!list_empty(&fault->deliver))
358 pollflags |= EPOLLIN | EPOLLRDNORM;
359 spin_unlock(&fault->lock);
360
361 return pollflags;
362 }
363
iommufd_fault_fops_release(struct inode * inode,struct file * filep)364 static int iommufd_fault_fops_release(struct inode *inode, struct file *filep)
365 {
366 struct iommufd_fault *fault = filep->private_data;
367
368 refcount_dec(&fault->obj.users);
369 iommufd_ctx_put(fault->ictx);
370 return 0;
371 }
372
373 static const struct file_operations iommufd_fault_fops = {
374 .owner = THIS_MODULE,
375 .open = nonseekable_open,
376 .read = iommufd_fault_fops_read,
377 .write = iommufd_fault_fops_write,
378 .poll = iommufd_fault_fops_poll,
379 .release = iommufd_fault_fops_release,
380 };
381
iommufd_fault_alloc(struct iommufd_ucmd * ucmd)382 int iommufd_fault_alloc(struct iommufd_ucmd *ucmd)
383 {
384 struct iommu_fault_alloc *cmd = ucmd->cmd;
385 struct iommufd_fault *fault;
386 struct file *filep;
387 int fdno;
388 int rc;
389
390 if (cmd->flags)
391 return -EOPNOTSUPP;
392
393 fault = iommufd_object_alloc(ucmd->ictx, fault, IOMMUFD_OBJ_FAULT);
394 if (IS_ERR(fault))
395 return PTR_ERR(fault);
396
397 fault->ictx = ucmd->ictx;
398 INIT_LIST_HEAD(&fault->deliver);
399 xa_init_flags(&fault->response, XA_FLAGS_ALLOC1);
400 mutex_init(&fault->mutex);
401 spin_lock_init(&fault->lock);
402 init_waitqueue_head(&fault->wait_queue);
403
404 filep = anon_inode_getfile("[iommufd-pgfault]", &iommufd_fault_fops,
405 fault, O_RDWR);
406 if (IS_ERR(filep)) {
407 rc = PTR_ERR(filep);
408 goto out_abort;
409 }
410
411 refcount_inc(&fault->obj.users);
412 iommufd_ctx_get(fault->ictx);
413 fault->filep = filep;
414
415 fdno = get_unused_fd_flags(O_CLOEXEC);
416 if (fdno < 0) {
417 rc = fdno;
418 goto out_abort;
419 }
420
421 cmd->out_fault_id = fault->obj.id;
422 cmd->out_fault_fd = fdno;
423
424 rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));
425 if (rc)
426 goto out_put_fdno;
427 iommufd_object_finalize(ucmd->ictx, &fault->obj);
428
429 fd_install(fdno, fault->filep);
430
431 return 0;
432 out_put_fdno:
433 put_unused_fd(fdno);
434 out_abort:
435 iommufd_object_abort_and_destroy(ucmd->ictx, &fault->obj);
436
437 return rc;
438 }
439
iommufd_fault_iopf_handler(struct iopf_group * group)440 int iommufd_fault_iopf_handler(struct iopf_group *group)
441 {
442 struct iommufd_hw_pagetable *hwpt;
443 struct iommufd_fault *fault;
444
445 hwpt = group->attach_handle->domain->fault_data;
446 fault = hwpt->fault;
447
448 spin_lock(&fault->lock);
449 list_add_tail(&group->node, &fault->deliver);
450 spin_unlock(&fault->lock);
451
452 wake_up_interruptible(&fault->wait_queue);
453
454 return 0;
455 }
456