1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2012 Red Hat, Inc. All rights reserved.
4 * Author: Alex Williamson <alex.williamson@redhat.com>
5 *
6 * Derived from original vfio:
7 * Copyright 2010 Cisco Systems, Inc. All rights reserved.
8 * Author: Tom Lyon, pugs@cisco.com
9 */
10
11 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
12
13 #include <linux/aperture.h>
14 #include <linux/device.h>
15 #include <linux/eventfd.h>
16 #include <linux/file.h>
17 #include <linux/interrupt.h>
18 #include <linux/iommu.h>
19 #include <linux/module.h>
20 #include <linux/mutex.h>
21 #include <linux/notifier.h>
22 #include <linux/pci.h>
23 #include <linux/pm_runtime.h>
24 #include <linux/slab.h>
25 #include <linux/types.h>
26 #include <linux/uaccess.h>
27 #include <linux/vgaarb.h>
28 #include <linux/nospec.h>
29 #include <linux/sched/mm.h>
30 #include <linux/iommufd.h>
31 #if IS_ENABLED(CONFIG_EEH)
32 #include <asm/eeh.h>
33 #endif
34
35 #include "vfio_pci_priv.h"
36
37 #define DRIVER_AUTHOR "Alex Williamson <alex.williamson@redhat.com>"
38 #define DRIVER_DESC "core driver for VFIO based PCI devices"
39
40 static bool nointxmask;
41 static bool disable_vga;
42 static bool disable_idle_d3;
43
44 /* List of PF's that vfio_pci_core_sriov_configure() has been called on */
45 static DEFINE_MUTEX(vfio_pci_sriov_pfs_mutex);
46 static LIST_HEAD(vfio_pci_sriov_pfs);
47
48 struct vfio_pci_dummy_resource {
49 struct resource resource;
50 int index;
51 struct list_head res_next;
52 };
53
54 struct vfio_pci_vf_token {
55 struct mutex lock;
56 uuid_t uuid;
57 int users;
58 };
59
60 struct vfio_pci_mmap_vma {
61 struct vm_area_struct *vma;
62 struct list_head vma_next;
63 };
64
vfio_vga_disabled(void)65 static inline bool vfio_vga_disabled(void)
66 {
67 #ifdef CONFIG_VFIO_PCI_VGA
68 return disable_vga;
69 #else
70 return true;
71 #endif
72 }
73
74 /*
75 * Our VGA arbiter participation is limited since we don't know anything
76 * about the device itself. However, if the device is the only VGA device
77 * downstream of a bridge and VFIO VGA support is disabled, then we can
78 * safely return legacy VGA IO and memory as not decoded since the user
79 * has no way to get to it and routing can be disabled externally at the
80 * bridge.
81 */
vfio_pci_set_decode(struct pci_dev * pdev,bool single_vga)82 static unsigned int vfio_pci_set_decode(struct pci_dev *pdev, bool single_vga)
83 {
84 struct pci_dev *tmp = NULL;
85 unsigned char max_busnr;
86 unsigned int decodes;
87
88 if (single_vga || !vfio_vga_disabled() || pci_is_root_bus(pdev->bus))
89 return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM |
90 VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM;
91
92 max_busnr = pci_bus_max_busnr(pdev->bus);
93 decodes = VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
94
95 while ((tmp = pci_get_class(PCI_CLASS_DISPLAY_VGA << 8, tmp)) != NULL) {
96 if (tmp == pdev ||
97 pci_domain_nr(tmp->bus) != pci_domain_nr(pdev->bus) ||
98 pci_is_root_bus(tmp->bus))
99 continue;
100
101 if (tmp->bus->number >= pdev->bus->number &&
102 tmp->bus->number <= max_busnr) {
103 pci_dev_put(tmp);
104 decodes |= VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM;
105 break;
106 }
107 }
108
109 return decodes;
110 }
111
vfio_pci_probe_mmaps(struct vfio_pci_core_device * vdev)112 static void vfio_pci_probe_mmaps(struct vfio_pci_core_device *vdev)
113 {
114 struct resource *res;
115 int i;
116 struct vfio_pci_dummy_resource *dummy_res;
117
118 for (i = 0; i < PCI_STD_NUM_BARS; i++) {
119 int bar = i + PCI_STD_RESOURCES;
120
121 res = &vdev->pdev->resource[bar];
122
123 if (!IS_ENABLED(CONFIG_VFIO_PCI_MMAP))
124 goto no_mmap;
125
126 if (!(res->flags & IORESOURCE_MEM))
127 goto no_mmap;
128
129 /*
130 * The PCI core shouldn't set up a resource with a
131 * type but zero size. But there may be bugs that
132 * cause us to do that.
133 */
134 if (!resource_size(res))
135 goto no_mmap;
136
137 if (resource_size(res) >= PAGE_SIZE) {
138 vdev->bar_mmap_supported[bar] = true;
139 continue;
140 }
141
142 if (!(res->start & ~PAGE_MASK)) {
143 /*
144 * Add a dummy resource to reserve the remainder
145 * of the exclusive page in case that hot-add
146 * device's bar is assigned into it.
147 */
148 dummy_res =
149 kzalloc(sizeof(*dummy_res), GFP_KERNEL_ACCOUNT);
150 if (dummy_res == NULL)
151 goto no_mmap;
152
153 dummy_res->resource.name = "vfio sub-page reserved";
154 dummy_res->resource.start = res->end + 1;
155 dummy_res->resource.end = res->start + PAGE_SIZE - 1;
156 dummy_res->resource.flags = res->flags;
157 if (request_resource(res->parent,
158 &dummy_res->resource)) {
159 kfree(dummy_res);
160 goto no_mmap;
161 }
162 dummy_res->index = bar;
163 list_add(&dummy_res->res_next,
164 &vdev->dummy_resources_list);
165 vdev->bar_mmap_supported[bar] = true;
166 continue;
167 }
168 /*
169 * Here we don't handle the case when the BAR is not page
170 * aligned because we can't expect the BAR will be
171 * assigned into the same location in a page in guest
172 * when we passthrough the BAR. And it's hard to access
173 * this BAR in userspace because we have no way to get
174 * the BAR's location in a page.
175 */
176 no_mmap:
177 vdev->bar_mmap_supported[bar] = false;
178 }
179 }
180
181 struct vfio_pci_group_info;
182 static void vfio_pci_dev_set_try_reset(struct vfio_device_set *dev_set);
183 static int vfio_pci_dev_set_hot_reset(struct vfio_device_set *dev_set,
184 struct vfio_pci_group_info *groups,
185 struct iommufd_ctx *iommufd_ctx);
186
187 /*
188 * INTx masking requires the ability to disable INTx signaling via PCI_COMMAND
189 * _and_ the ability detect when the device is asserting INTx via PCI_STATUS.
190 * If a device implements the former but not the latter we would typically
191 * expect broken_intx_masking be set and require an exclusive interrupt.
192 * However since we do have control of the device's ability to assert INTx,
193 * we can instead pretend that the device does not implement INTx, virtualizing
194 * the pin register to report zero and maintaining DisINTx set on the host.
195 */
vfio_pci_nointx(struct pci_dev * pdev)196 static bool vfio_pci_nointx(struct pci_dev *pdev)
197 {
198 switch (pdev->vendor) {
199 case PCI_VENDOR_ID_INTEL:
200 switch (pdev->device) {
201 /* All i40e (XL710/X710/XXV710) 10/20/25/40GbE NICs */
202 case 0x1572:
203 case 0x1574:
204 case 0x1580 ... 0x1581:
205 case 0x1583 ... 0x158b:
206 case 0x37d0 ... 0x37d2:
207 /* X550 */
208 case 0x1563:
209 return true;
210 default:
211 return false;
212 }
213 }
214
215 return false;
216 }
217
vfio_pci_probe_power_state(struct vfio_pci_core_device * vdev)218 static void vfio_pci_probe_power_state(struct vfio_pci_core_device *vdev)
219 {
220 struct pci_dev *pdev = vdev->pdev;
221 u16 pmcsr;
222
223 if (!pdev->pm_cap)
224 return;
225
226 pci_read_config_word(pdev, pdev->pm_cap + PCI_PM_CTRL, &pmcsr);
227
228 vdev->needs_pm_restore = !(pmcsr & PCI_PM_CTRL_NO_SOFT_RESET);
229 }
230
231 /*
232 * pci_set_power_state() wrapper handling devices which perform a soft reset on
233 * D3->D0 transition. Save state prior to D0/1/2->D3, stash it on the vdev,
234 * restore when returned to D0. Saved separately from pci_saved_state for use
235 * by PM capability emulation and separately from pci_dev internal saved state
236 * to avoid it being overwritten and consumed around other resets.
237 */
vfio_pci_set_power_state(struct vfio_pci_core_device * vdev,pci_power_t state)238 int vfio_pci_set_power_state(struct vfio_pci_core_device *vdev, pci_power_t state)
239 {
240 struct pci_dev *pdev = vdev->pdev;
241 bool needs_restore = false, needs_save = false;
242 int ret;
243
244 /* Prevent changing power state for PFs with VFs enabled */
245 if (pci_num_vf(pdev) && state > PCI_D0)
246 return -EBUSY;
247
248 if (vdev->needs_pm_restore) {
249 if (pdev->current_state < PCI_D3hot && state >= PCI_D3hot) {
250 pci_save_state(pdev);
251 needs_save = true;
252 }
253
254 if (pdev->current_state >= PCI_D3hot && state <= PCI_D0)
255 needs_restore = true;
256 }
257
258 ret = pci_set_power_state(pdev, state);
259
260 if (!ret) {
261 /* D3 might be unsupported via quirk, skip unless in D3 */
262 if (needs_save && pdev->current_state >= PCI_D3hot) {
263 /*
264 * The current PCI state will be saved locally in
265 * 'pm_save' during the D3hot transition. When the
266 * device state is changed to D0 again with the current
267 * function, then pci_store_saved_state() will restore
268 * the state and will free the memory pointed by
269 * 'pm_save'. There are few cases where the PCI power
270 * state can be changed to D0 without the involvement
271 * of the driver. For these cases, free the earlier
272 * allocated memory first before overwriting 'pm_save'
273 * to prevent the memory leak.
274 */
275 kfree(vdev->pm_save);
276 vdev->pm_save = pci_store_saved_state(pdev);
277 } else if (needs_restore) {
278 pci_load_and_free_saved_state(pdev, &vdev->pm_save);
279 pci_restore_state(pdev);
280 }
281 }
282
283 return ret;
284 }
285
vfio_pci_runtime_pm_entry(struct vfio_pci_core_device * vdev,struct eventfd_ctx * efdctx)286 static int vfio_pci_runtime_pm_entry(struct vfio_pci_core_device *vdev,
287 struct eventfd_ctx *efdctx)
288 {
289 /*
290 * The vdev power related flags are protected with 'memory_lock'
291 * semaphore.
292 */
293 vfio_pci_zap_and_down_write_memory_lock(vdev);
294 if (vdev->pm_runtime_engaged) {
295 up_write(&vdev->memory_lock);
296 return -EINVAL;
297 }
298
299 vdev->pm_runtime_engaged = true;
300 vdev->pm_wake_eventfd_ctx = efdctx;
301 pm_runtime_put_noidle(&vdev->pdev->dev);
302 up_write(&vdev->memory_lock);
303
304 return 0;
305 }
306
vfio_pci_core_pm_entry(struct vfio_device * device,u32 flags,void __user * arg,size_t argsz)307 static int vfio_pci_core_pm_entry(struct vfio_device *device, u32 flags,
308 void __user *arg, size_t argsz)
309 {
310 struct vfio_pci_core_device *vdev =
311 container_of(device, struct vfio_pci_core_device, vdev);
312 int ret;
313
314 ret = vfio_check_feature(flags, argsz, VFIO_DEVICE_FEATURE_SET, 0);
315 if (ret != 1)
316 return ret;
317
318 /*
319 * Inside vfio_pci_runtime_pm_entry(), only the runtime PM usage count
320 * will be decremented. The pm_runtime_put() will be invoked again
321 * while returning from the ioctl and then the device can go into
322 * runtime suspended state.
323 */
324 return vfio_pci_runtime_pm_entry(vdev, NULL);
325 }
326
vfio_pci_core_pm_entry_with_wakeup(struct vfio_device * device,u32 flags,struct vfio_device_low_power_entry_with_wakeup __user * arg,size_t argsz)327 static int vfio_pci_core_pm_entry_with_wakeup(
328 struct vfio_device *device, u32 flags,
329 struct vfio_device_low_power_entry_with_wakeup __user *arg,
330 size_t argsz)
331 {
332 struct vfio_pci_core_device *vdev =
333 container_of(device, struct vfio_pci_core_device, vdev);
334 struct vfio_device_low_power_entry_with_wakeup entry;
335 struct eventfd_ctx *efdctx;
336 int ret;
337
338 ret = vfio_check_feature(flags, argsz, VFIO_DEVICE_FEATURE_SET,
339 sizeof(entry));
340 if (ret != 1)
341 return ret;
342
343 if (copy_from_user(&entry, arg, sizeof(entry)))
344 return -EFAULT;
345
346 if (entry.wakeup_eventfd < 0)
347 return -EINVAL;
348
349 efdctx = eventfd_ctx_fdget(entry.wakeup_eventfd);
350 if (IS_ERR(efdctx))
351 return PTR_ERR(efdctx);
352
353 ret = vfio_pci_runtime_pm_entry(vdev, efdctx);
354 if (ret)
355 eventfd_ctx_put(efdctx);
356
357 return ret;
358 }
359
__vfio_pci_runtime_pm_exit(struct vfio_pci_core_device * vdev)360 static void __vfio_pci_runtime_pm_exit(struct vfio_pci_core_device *vdev)
361 {
362 if (vdev->pm_runtime_engaged) {
363 vdev->pm_runtime_engaged = false;
364 pm_runtime_get_noresume(&vdev->pdev->dev);
365
366 if (vdev->pm_wake_eventfd_ctx) {
367 eventfd_ctx_put(vdev->pm_wake_eventfd_ctx);
368 vdev->pm_wake_eventfd_ctx = NULL;
369 }
370 }
371 }
372
vfio_pci_runtime_pm_exit(struct vfio_pci_core_device * vdev)373 static void vfio_pci_runtime_pm_exit(struct vfio_pci_core_device *vdev)
374 {
375 /*
376 * The vdev power related flags are protected with 'memory_lock'
377 * semaphore.
378 */
379 down_write(&vdev->memory_lock);
380 __vfio_pci_runtime_pm_exit(vdev);
381 up_write(&vdev->memory_lock);
382 }
383
vfio_pci_core_pm_exit(struct vfio_device * device,u32 flags,void __user * arg,size_t argsz)384 static int vfio_pci_core_pm_exit(struct vfio_device *device, u32 flags,
385 void __user *arg, size_t argsz)
386 {
387 struct vfio_pci_core_device *vdev =
388 container_of(device, struct vfio_pci_core_device, vdev);
389 int ret;
390
391 ret = vfio_check_feature(flags, argsz, VFIO_DEVICE_FEATURE_SET, 0);
392 if (ret != 1)
393 return ret;
394
395 /*
396 * The device is always in the active state here due to pm wrappers
397 * around ioctls. If the device had entered a low power state and
398 * pm_wake_eventfd_ctx is valid, vfio_pci_core_runtime_resume() has
399 * already signaled the eventfd and exited low power mode itself.
400 * pm_runtime_engaged protects the redundant call here.
401 */
402 vfio_pci_runtime_pm_exit(vdev);
403 return 0;
404 }
405
406 #ifdef CONFIG_PM
vfio_pci_core_runtime_suspend(struct device * dev)407 static int vfio_pci_core_runtime_suspend(struct device *dev)
408 {
409 struct vfio_pci_core_device *vdev = dev_get_drvdata(dev);
410
411 down_write(&vdev->memory_lock);
412 /*
413 * The user can move the device into D3hot state before invoking
414 * power management IOCTL. Move the device into D0 state here and then
415 * the pci-driver core runtime PM suspend function will move the device
416 * into the low power state. Also, for the devices which have
417 * NoSoftRst-, it will help in restoring the original state
418 * (saved locally in 'vdev->pm_save').
419 */
420 vfio_pci_set_power_state(vdev, PCI_D0);
421 up_write(&vdev->memory_lock);
422
423 /*
424 * If INTx is enabled, then mask INTx before going into the runtime
425 * suspended state and unmask the same in the runtime resume.
426 * If INTx has already been masked by the user, then
427 * vfio_pci_intx_mask() will return false and in that case, INTx
428 * should not be unmasked in the runtime resume.
429 */
430 vdev->pm_intx_masked = ((vdev->irq_type == VFIO_PCI_INTX_IRQ_INDEX) &&
431 vfio_pci_intx_mask(vdev));
432
433 return 0;
434 }
435
vfio_pci_core_runtime_resume(struct device * dev)436 static int vfio_pci_core_runtime_resume(struct device *dev)
437 {
438 struct vfio_pci_core_device *vdev = dev_get_drvdata(dev);
439
440 /*
441 * Resume with a pm_wake_eventfd_ctx signals the eventfd and exit
442 * low power mode.
443 */
444 down_write(&vdev->memory_lock);
445 if (vdev->pm_wake_eventfd_ctx) {
446 eventfd_signal(vdev->pm_wake_eventfd_ctx, 1);
447 __vfio_pci_runtime_pm_exit(vdev);
448 }
449 up_write(&vdev->memory_lock);
450
451 if (vdev->pm_intx_masked)
452 vfio_pci_intx_unmask(vdev);
453
454 return 0;
455 }
456 #endif /* CONFIG_PM */
457
458 /*
459 * The pci-driver core runtime PM routines always save the device state
460 * before going into suspended state. If the device is going into low power
461 * state with only with runtime PM ops, then no explicit handling is needed
462 * for the devices which have NoSoftRst-.
463 */
464 static const struct dev_pm_ops vfio_pci_core_pm_ops = {
465 SET_RUNTIME_PM_OPS(vfio_pci_core_runtime_suspend,
466 vfio_pci_core_runtime_resume,
467 NULL)
468 };
469
vfio_pci_core_enable(struct vfio_pci_core_device * vdev)470 int vfio_pci_core_enable(struct vfio_pci_core_device *vdev)
471 {
472 struct pci_dev *pdev = vdev->pdev;
473 int ret;
474 u16 cmd;
475 u8 msix_pos;
476
477 if (!disable_idle_d3) {
478 ret = pm_runtime_resume_and_get(&pdev->dev);
479 if (ret < 0)
480 return ret;
481 }
482
483 /* Don't allow our initial saved state to include busmaster */
484 pci_clear_master(pdev);
485
486 ret = pci_enable_device(pdev);
487 if (ret)
488 goto out_power;
489
490 /* If reset fails because of the device lock, fail this path entirely */
491 ret = pci_try_reset_function(pdev);
492 if (ret == -EAGAIN)
493 goto out_disable_device;
494
495 vdev->reset_works = !ret;
496 pci_save_state(pdev);
497 vdev->pci_saved_state = pci_store_saved_state(pdev);
498 if (!vdev->pci_saved_state)
499 pci_dbg(pdev, "%s: Couldn't store saved state\n", __func__);
500
501 if (likely(!nointxmask)) {
502 if (vfio_pci_nointx(pdev)) {
503 pci_info(pdev, "Masking broken INTx support\n");
504 vdev->nointx = true;
505 pci_intx(pdev, 0);
506 } else
507 vdev->pci_2_3 = pci_intx_mask_supported(pdev);
508 }
509
510 pci_read_config_word(pdev, PCI_COMMAND, &cmd);
511 if (vdev->pci_2_3 && (cmd & PCI_COMMAND_INTX_DISABLE)) {
512 cmd &= ~PCI_COMMAND_INTX_DISABLE;
513 pci_write_config_word(pdev, PCI_COMMAND, cmd);
514 }
515
516 ret = vfio_pci_zdev_open_device(vdev);
517 if (ret)
518 goto out_free_state;
519
520 ret = vfio_config_init(vdev);
521 if (ret)
522 goto out_free_zdev;
523
524 msix_pos = pdev->msix_cap;
525 if (msix_pos) {
526 u16 flags;
527 u32 table;
528
529 pci_read_config_word(pdev, msix_pos + PCI_MSIX_FLAGS, &flags);
530 pci_read_config_dword(pdev, msix_pos + PCI_MSIX_TABLE, &table);
531
532 vdev->msix_bar = table & PCI_MSIX_TABLE_BIR;
533 vdev->msix_offset = table & PCI_MSIX_TABLE_OFFSET;
534 vdev->msix_size = ((flags & PCI_MSIX_FLAGS_QSIZE) + 1) * 16;
535 vdev->has_dyn_msix = pci_msix_can_alloc_dyn(pdev);
536 } else {
537 vdev->msix_bar = 0xFF;
538 vdev->has_dyn_msix = false;
539 }
540
541 if (!vfio_vga_disabled() && vfio_pci_is_vga(pdev))
542 vdev->has_vga = true;
543
544
545 return 0;
546
547 out_free_zdev:
548 vfio_pci_zdev_close_device(vdev);
549 out_free_state:
550 kfree(vdev->pci_saved_state);
551 vdev->pci_saved_state = NULL;
552 out_disable_device:
553 pci_disable_device(pdev);
554 out_power:
555 if (!disable_idle_d3)
556 pm_runtime_put(&pdev->dev);
557 return ret;
558 }
559 EXPORT_SYMBOL_GPL(vfio_pci_core_enable);
560
vfio_pci_core_disable(struct vfio_pci_core_device * vdev)561 void vfio_pci_core_disable(struct vfio_pci_core_device *vdev)
562 {
563 struct pci_dev *pdev = vdev->pdev;
564 struct vfio_pci_dummy_resource *dummy_res, *tmp;
565 struct vfio_pci_ioeventfd *ioeventfd, *ioeventfd_tmp;
566 int i, bar;
567
568 /* For needs_reset */
569 lockdep_assert_held(&vdev->vdev.dev_set->lock);
570
571 /*
572 * This function can be invoked while the power state is non-D0.
573 * This non-D0 power state can be with or without runtime PM.
574 * vfio_pci_runtime_pm_exit() will internally increment the usage
575 * count corresponding to pm_runtime_put() called during low power
576 * feature entry and then pm_runtime_resume() will wake up the device,
577 * if the device has already gone into the suspended state. Otherwise,
578 * the vfio_pci_set_power_state() will change the device power state
579 * to D0.
580 */
581 vfio_pci_runtime_pm_exit(vdev);
582 pm_runtime_resume(&pdev->dev);
583
584 /*
585 * This function calls __pci_reset_function_locked() which internally
586 * can use pci_pm_reset() for the function reset. pci_pm_reset() will
587 * fail if the power state is non-D0. Also, for the devices which
588 * have NoSoftRst-, the reset function can cause the PCI config space
589 * reset without restoring the original state (saved locally in
590 * 'vdev->pm_save').
591 */
592 vfio_pci_set_power_state(vdev, PCI_D0);
593
594 /* Stop the device from further DMA */
595 pci_clear_master(pdev);
596
597 vfio_pci_set_irqs_ioctl(vdev, VFIO_IRQ_SET_DATA_NONE |
598 VFIO_IRQ_SET_ACTION_TRIGGER,
599 vdev->irq_type, 0, 0, NULL);
600
601 /* Device closed, don't need mutex here */
602 list_for_each_entry_safe(ioeventfd, ioeventfd_tmp,
603 &vdev->ioeventfds_list, next) {
604 vfio_virqfd_disable(&ioeventfd->virqfd);
605 list_del(&ioeventfd->next);
606 kfree(ioeventfd);
607 }
608 vdev->ioeventfds_nr = 0;
609
610 vdev->virq_disabled = false;
611
612 for (i = 0; i < vdev->num_regions; i++)
613 vdev->region[i].ops->release(vdev, &vdev->region[i]);
614
615 vdev->num_regions = 0;
616 kfree(vdev->region);
617 vdev->region = NULL; /* don't krealloc a freed pointer */
618
619 vfio_config_free(vdev);
620
621 for (i = 0; i < PCI_STD_NUM_BARS; i++) {
622 bar = i + PCI_STD_RESOURCES;
623 if (!vdev->barmap[bar])
624 continue;
625 pci_iounmap(pdev, vdev->barmap[bar]);
626 pci_release_selected_regions(pdev, 1 << bar);
627 vdev->barmap[bar] = NULL;
628 }
629
630 list_for_each_entry_safe(dummy_res, tmp,
631 &vdev->dummy_resources_list, res_next) {
632 list_del(&dummy_res->res_next);
633 release_resource(&dummy_res->resource);
634 kfree(dummy_res);
635 }
636
637 vdev->needs_reset = true;
638
639 vfio_pci_zdev_close_device(vdev);
640
641 /*
642 * If we have saved state, restore it. If we can reset the device,
643 * even better. Resetting with current state seems better than
644 * nothing, but saving and restoring current state without reset
645 * is just busy work.
646 */
647 if (pci_load_and_free_saved_state(pdev, &vdev->pci_saved_state)) {
648 pci_info(pdev, "%s: Couldn't reload saved state\n", __func__);
649
650 if (!vdev->reset_works)
651 goto out;
652
653 pci_save_state(pdev);
654 }
655
656 /*
657 * Disable INTx and MSI, presumably to avoid spurious interrupts
658 * during reset. Stolen from pci_reset_function()
659 */
660 pci_write_config_word(pdev, PCI_COMMAND, PCI_COMMAND_INTX_DISABLE);
661
662 /*
663 * Try to get the locks ourselves to prevent a deadlock. The
664 * success of this is dependent on being able to lock the device,
665 * which is not always possible.
666 * We can not use the "try" reset interface here, which will
667 * overwrite the previously restored configuration information.
668 */
669 if (vdev->reset_works && pci_dev_trylock(pdev)) {
670 if (!__pci_reset_function_locked(pdev))
671 vdev->needs_reset = false;
672 pci_dev_unlock(pdev);
673 }
674
675 pci_restore_state(pdev);
676 out:
677 pci_disable_device(pdev);
678
679 vfio_pci_dev_set_try_reset(vdev->vdev.dev_set);
680
681 /* Put the pm-runtime usage counter acquired during enable */
682 if (!disable_idle_d3)
683 pm_runtime_put(&pdev->dev);
684 }
685 EXPORT_SYMBOL_GPL(vfio_pci_core_disable);
686
vfio_pci_core_close_device(struct vfio_device * core_vdev)687 void vfio_pci_core_close_device(struct vfio_device *core_vdev)
688 {
689 struct vfio_pci_core_device *vdev =
690 container_of(core_vdev, struct vfio_pci_core_device, vdev);
691
692 if (vdev->sriov_pf_core_dev) {
693 mutex_lock(&vdev->sriov_pf_core_dev->vf_token->lock);
694 WARN_ON(!vdev->sriov_pf_core_dev->vf_token->users);
695 vdev->sriov_pf_core_dev->vf_token->users--;
696 mutex_unlock(&vdev->sriov_pf_core_dev->vf_token->lock);
697 }
698 #if IS_ENABLED(CONFIG_EEH)
699 eeh_dev_release(vdev->pdev);
700 #endif
701 vfio_pci_core_disable(vdev);
702
703 mutex_lock(&vdev->igate);
704 if (vdev->err_trigger) {
705 eventfd_ctx_put(vdev->err_trigger);
706 vdev->err_trigger = NULL;
707 }
708 if (vdev->req_trigger) {
709 eventfd_ctx_put(vdev->req_trigger);
710 vdev->req_trigger = NULL;
711 }
712 mutex_unlock(&vdev->igate);
713 }
714 EXPORT_SYMBOL_GPL(vfio_pci_core_close_device);
715
vfio_pci_core_finish_enable(struct vfio_pci_core_device * vdev)716 void vfio_pci_core_finish_enable(struct vfio_pci_core_device *vdev)
717 {
718 vfio_pci_probe_mmaps(vdev);
719 #if IS_ENABLED(CONFIG_EEH)
720 eeh_dev_open(vdev->pdev);
721 #endif
722
723 if (vdev->sriov_pf_core_dev) {
724 mutex_lock(&vdev->sriov_pf_core_dev->vf_token->lock);
725 vdev->sriov_pf_core_dev->vf_token->users++;
726 mutex_unlock(&vdev->sriov_pf_core_dev->vf_token->lock);
727 }
728 }
729 EXPORT_SYMBOL_GPL(vfio_pci_core_finish_enable);
730
vfio_pci_get_irq_count(struct vfio_pci_core_device * vdev,int irq_type)731 static int vfio_pci_get_irq_count(struct vfio_pci_core_device *vdev, int irq_type)
732 {
733 if (irq_type == VFIO_PCI_INTX_IRQ_INDEX) {
734 u8 pin;
735
736 if (!IS_ENABLED(CONFIG_VFIO_PCI_INTX) ||
737 vdev->nointx || vdev->pdev->is_virtfn)
738 return 0;
739
740 pci_read_config_byte(vdev->pdev, PCI_INTERRUPT_PIN, &pin);
741
742 return pin ? 1 : 0;
743 } else if (irq_type == VFIO_PCI_MSI_IRQ_INDEX) {
744 u8 pos;
745 u16 flags;
746
747 pos = vdev->pdev->msi_cap;
748 if (pos) {
749 pci_read_config_word(vdev->pdev,
750 pos + PCI_MSI_FLAGS, &flags);
751 return 1 << ((flags & PCI_MSI_FLAGS_QMASK) >> 1);
752 }
753 } else if (irq_type == VFIO_PCI_MSIX_IRQ_INDEX) {
754 u8 pos;
755 u16 flags;
756
757 pos = vdev->pdev->msix_cap;
758 if (pos) {
759 pci_read_config_word(vdev->pdev,
760 pos + PCI_MSIX_FLAGS, &flags);
761
762 return (flags & PCI_MSIX_FLAGS_QSIZE) + 1;
763 }
764 } else if (irq_type == VFIO_PCI_ERR_IRQ_INDEX) {
765 if (pci_is_pcie(vdev->pdev))
766 return 1;
767 } else if (irq_type == VFIO_PCI_REQ_IRQ_INDEX) {
768 return 1;
769 }
770
771 return 0;
772 }
773
vfio_pci_count_devs(struct pci_dev * pdev,void * data)774 static int vfio_pci_count_devs(struct pci_dev *pdev, void *data)
775 {
776 (*(int *)data)++;
777 return 0;
778 }
779
780 struct vfio_pci_fill_info {
781 struct vfio_device *vdev;
782 struct vfio_pci_dependent_device *devices;
783 int nr_devices;
784 u32 count;
785 u32 flags;
786 };
787
vfio_pci_fill_devs(struct pci_dev * pdev,void * data)788 static int vfio_pci_fill_devs(struct pci_dev *pdev, void *data)
789 {
790 struct vfio_pci_dependent_device *info;
791 struct vfio_pci_fill_info *fill = data;
792
793 /* The topology changed since we counted devices */
794 if (fill->count >= fill->nr_devices)
795 return -EAGAIN;
796
797 info = &fill->devices[fill->count++];
798 info->segment = pci_domain_nr(pdev->bus);
799 info->bus = pdev->bus->number;
800 info->devfn = pdev->devfn;
801
802 if (fill->flags & VFIO_PCI_HOT_RESET_FLAG_DEV_ID) {
803 struct iommufd_ctx *iommufd = vfio_iommufd_device_ictx(fill->vdev);
804 struct vfio_device_set *dev_set = fill->vdev->dev_set;
805 struct vfio_device *vdev;
806
807 /*
808 * hot-reset requires all affected devices be represented in
809 * the dev_set.
810 */
811 vdev = vfio_find_device_in_devset(dev_set, &pdev->dev);
812 if (!vdev) {
813 info->devid = VFIO_PCI_DEVID_NOT_OWNED;
814 } else {
815 int id = vfio_iommufd_get_dev_id(vdev, iommufd);
816
817 if (id > 0)
818 info->devid = id;
819 else if (id == -ENOENT)
820 info->devid = VFIO_PCI_DEVID_OWNED;
821 else
822 info->devid = VFIO_PCI_DEVID_NOT_OWNED;
823 }
824 /* If devid is VFIO_PCI_DEVID_NOT_OWNED, clear owned flag. */
825 if (info->devid == VFIO_PCI_DEVID_NOT_OWNED)
826 fill->flags &= ~VFIO_PCI_HOT_RESET_FLAG_DEV_ID_OWNED;
827 } else {
828 struct iommu_group *iommu_group;
829
830 iommu_group = iommu_group_get(&pdev->dev);
831 if (!iommu_group)
832 return -EPERM; /* Cannot reset non-isolated devices */
833
834 info->group_id = iommu_group_id(iommu_group);
835 iommu_group_put(iommu_group);
836 }
837
838 return 0;
839 }
840
841 struct vfio_pci_group_info {
842 int count;
843 struct file **files;
844 };
845
vfio_pci_dev_below_slot(struct pci_dev * pdev,struct pci_slot * slot)846 static bool vfio_pci_dev_below_slot(struct pci_dev *pdev, struct pci_slot *slot)
847 {
848 for (; pdev; pdev = pdev->bus->self)
849 if (pdev->bus == slot->bus)
850 return (pdev->slot == slot);
851 return false;
852 }
853
854 struct vfio_pci_walk_info {
855 int (*fn)(struct pci_dev *pdev, void *data);
856 void *data;
857 struct pci_dev *pdev;
858 bool slot;
859 int ret;
860 };
861
vfio_pci_walk_wrapper(struct pci_dev * pdev,void * data)862 static int vfio_pci_walk_wrapper(struct pci_dev *pdev, void *data)
863 {
864 struct vfio_pci_walk_info *walk = data;
865
866 if (!walk->slot || vfio_pci_dev_below_slot(pdev, walk->pdev->slot))
867 walk->ret = walk->fn(pdev, walk->data);
868
869 return walk->ret;
870 }
871
vfio_pci_for_each_slot_or_bus(struct pci_dev * pdev,int (* fn)(struct pci_dev *,void * data),void * data,bool slot)872 static int vfio_pci_for_each_slot_or_bus(struct pci_dev *pdev,
873 int (*fn)(struct pci_dev *,
874 void *data), void *data,
875 bool slot)
876 {
877 struct vfio_pci_walk_info walk = {
878 .fn = fn, .data = data, .pdev = pdev, .slot = slot, .ret = 0,
879 };
880
881 pci_walk_bus(pdev->bus, vfio_pci_walk_wrapper, &walk);
882
883 return walk.ret;
884 }
885
msix_mmappable_cap(struct vfio_pci_core_device * vdev,struct vfio_info_cap * caps)886 static int msix_mmappable_cap(struct vfio_pci_core_device *vdev,
887 struct vfio_info_cap *caps)
888 {
889 struct vfio_info_cap_header header = {
890 .id = VFIO_REGION_INFO_CAP_MSIX_MAPPABLE,
891 .version = 1
892 };
893
894 return vfio_info_add_capability(caps, &header, sizeof(header));
895 }
896
vfio_pci_core_register_dev_region(struct vfio_pci_core_device * vdev,unsigned int type,unsigned int subtype,const struct vfio_pci_regops * ops,size_t size,u32 flags,void * data)897 int vfio_pci_core_register_dev_region(struct vfio_pci_core_device *vdev,
898 unsigned int type, unsigned int subtype,
899 const struct vfio_pci_regops *ops,
900 size_t size, u32 flags, void *data)
901 {
902 struct vfio_pci_region *region;
903
904 region = krealloc(vdev->region,
905 (vdev->num_regions + 1) * sizeof(*region),
906 GFP_KERNEL_ACCOUNT);
907 if (!region)
908 return -ENOMEM;
909
910 vdev->region = region;
911 vdev->region[vdev->num_regions].type = type;
912 vdev->region[vdev->num_regions].subtype = subtype;
913 vdev->region[vdev->num_regions].ops = ops;
914 vdev->region[vdev->num_regions].size = size;
915 vdev->region[vdev->num_regions].flags = flags;
916 vdev->region[vdev->num_regions].data = data;
917
918 vdev->num_regions++;
919
920 return 0;
921 }
922 EXPORT_SYMBOL_GPL(vfio_pci_core_register_dev_region);
923
vfio_pci_info_atomic_cap(struct vfio_pci_core_device * vdev,struct vfio_info_cap * caps)924 static int vfio_pci_info_atomic_cap(struct vfio_pci_core_device *vdev,
925 struct vfio_info_cap *caps)
926 {
927 struct vfio_device_info_cap_pci_atomic_comp cap = {
928 .header.id = VFIO_DEVICE_INFO_CAP_PCI_ATOMIC_COMP,
929 .header.version = 1
930 };
931 struct pci_dev *pdev = pci_physfn(vdev->pdev);
932 u32 devcap2;
933
934 pcie_capability_read_dword(pdev, PCI_EXP_DEVCAP2, &devcap2);
935
936 if ((devcap2 & PCI_EXP_DEVCAP2_ATOMIC_COMP32) &&
937 !pci_enable_atomic_ops_to_root(pdev, PCI_EXP_DEVCAP2_ATOMIC_COMP32))
938 cap.flags |= VFIO_PCI_ATOMIC_COMP32;
939
940 if ((devcap2 & PCI_EXP_DEVCAP2_ATOMIC_COMP64) &&
941 !pci_enable_atomic_ops_to_root(pdev, PCI_EXP_DEVCAP2_ATOMIC_COMP64))
942 cap.flags |= VFIO_PCI_ATOMIC_COMP64;
943
944 if ((devcap2 & PCI_EXP_DEVCAP2_ATOMIC_COMP128) &&
945 !pci_enable_atomic_ops_to_root(pdev,
946 PCI_EXP_DEVCAP2_ATOMIC_COMP128))
947 cap.flags |= VFIO_PCI_ATOMIC_COMP128;
948
949 if (!cap.flags)
950 return -ENODEV;
951
952 return vfio_info_add_capability(caps, &cap.header, sizeof(cap));
953 }
954
vfio_pci_ioctl_get_info(struct vfio_pci_core_device * vdev,struct vfio_device_info __user * arg)955 static int vfio_pci_ioctl_get_info(struct vfio_pci_core_device *vdev,
956 struct vfio_device_info __user *arg)
957 {
958 unsigned long minsz = offsetofend(struct vfio_device_info, num_irqs);
959 struct vfio_device_info info = {};
960 struct vfio_info_cap caps = { .buf = NULL, .size = 0 };
961 int ret;
962
963 if (copy_from_user(&info, arg, minsz))
964 return -EFAULT;
965
966 if (info.argsz < minsz)
967 return -EINVAL;
968
969 minsz = min_t(size_t, info.argsz, sizeof(info));
970
971 info.flags = VFIO_DEVICE_FLAGS_PCI;
972
973 if (vdev->reset_works)
974 info.flags |= VFIO_DEVICE_FLAGS_RESET;
975
976 info.num_regions = VFIO_PCI_NUM_REGIONS + vdev->num_regions;
977 info.num_irqs = VFIO_PCI_NUM_IRQS;
978
979 ret = vfio_pci_info_zdev_add_caps(vdev, &caps);
980 if (ret && ret != -ENODEV) {
981 pci_warn(vdev->pdev,
982 "Failed to setup zPCI info capabilities\n");
983 return ret;
984 }
985
986 ret = vfio_pci_info_atomic_cap(vdev, &caps);
987 if (ret && ret != -ENODEV) {
988 pci_warn(vdev->pdev,
989 "Failed to setup AtomicOps info capability\n");
990 return ret;
991 }
992
993 if (caps.size) {
994 info.flags |= VFIO_DEVICE_FLAGS_CAPS;
995 if (info.argsz < sizeof(info) + caps.size) {
996 info.argsz = sizeof(info) + caps.size;
997 } else {
998 vfio_info_cap_shift(&caps, sizeof(info));
999 if (copy_to_user(arg + 1, caps.buf, caps.size)) {
1000 kfree(caps.buf);
1001 return -EFAULT;
1002 }
1003 info.cap_offset = sizeof(*arg);
1004 }
1005
1006 kfree(caps.buf);
1007 }
1008
1009 return copy_to_user(arg, &info, minsz) ? -EFAULT : 0;
1010 }
1011
vfio_pci_ioctl_get_region_info(struct vfio_pci_core_device * vdev,struct vfio_region_info __user * arg)1012 static int vfio_pci_ioctl_get_region_info(struct vfio_pci_core_device *vdev,
1013 struct vfio_region_info __user *arg)
1014 {
1015 unsigned long minsz = offsetofend(struct vfio_region_info, offset);
1016 struct pci_dev *pdev = vdev->pdev;
1017 struct vfio_region_info info;
1018 struct vfio_info_cap caps = { .buf = NULL, .size = 0 };
1019 int i, ret;
1020
1021 if (copy_from_user(&info, arg, minsz))
1022 return -EFAULT;
1023
1024 if (info.argsz < minsz)
1025 return -EINVAL;
1026
1027 switch (info.index) {
1028 case VFIO_PCI_CONFIG_REGION_INDEX:
1029 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
1030 info.size = pdev->cfg_size;
1031 info.flags = VFIO_REGION_INFO_FLAG_READ |
1032 VFIO_REGION_INFO_FLAG_WRITE;
1033 break;
1034 case VFIO_PCI_BAR0_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX:
1035 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
1036 info.size = pci_resource_len(pdev, info.index);
1037 if (!info.size) {
1038 info.flags = 0;
1039 break;
1040 }
1041
1042 info.flags = VFIO_REGION_INFO_FLAG_READ |
1043 VFIO_REGION_INFO_FLAG_WRITE;
1044 if (vdev->bar_mmap_supported[info.index]) {
1045 info.flags |= VFIO_REGION_INFO_FLAG_MMAP;
1046 if (info.index == vdev->msix_bar) {
1047 ret = msix_mmappable_cap(vdev, &caps);
1048 if (ret)
1049 return ret;
1050 }
1051 }
1052
1053 break;
1054 case VFIO_PCI_ROM_REGION_INDEX: {
1055 void __iomem *io;
1056 size_t size;
1057 u16 cmd;
1058
1059 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
1060 info.flags = 0;
1061
1062 /* Report the BAR size, not the ROM size */
1063 info.size = pci_resource_len(pdev, info.index);
1064 if (!info.size) {
1065 /* Shadow ROMs appear as PCI option ROMs */
1066 if (pdev->resource[PCI_ROM_RESOURCE].flags &
1067 IORESOURCE_ROM_SHADOW)
1068 info.size = 0x20000;
1069 else
1070 break;
1071 }
1072
1073 /*
1074 * Is it really there? Enable memory decode for implicit access
1075 * in pci_map_rom().
1076 */
1077 cmd = vfio_pci_memory_lock_and_enable(vdev);
1078 io = pci_map_rom(pdev, &size);
1079 if (io) {
1080 info.flags = VFIO_REGION_INFO_FLAG_READ;
1081 pci_unmap_rom(pdev, io);
1082 } else {
1083 info.size = 0;
1084 }
1085 vfio_pci_memory_unlock_and_restore(vdev, cmd);
1086
1087 break;
1088 }
1089 case VFIO_PCI_VGA_REGION_INDEX:
1090 if (!vdev->has_vga)
1091 return -EINVAL;
1092
1093 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
1094 info.size = 0xc0000;
1095 info.flags = VFIO_REGION_INFO_FLAG_READ |
1096 VFIO_REGION_INFO_FLAG_WRITE;
1097
1098 break;
1099 default: {
1100 struct vfio_region_info_cap_type cap_type = {
1101 .header.id = VFIO_REGION_INFO_CAP_TYPE,
1102 .header.version = 1
1103 };
1104
1105 if (info.index >= VFIO_PCI_NUM_REGIONS + vdev->num_regions)
1106 return -EINVAL;
1107 info.index = array_index_nospec(
1108 info.index, VFIO_PCI_NUM_REGIONS + vdev->num_regions);
1109
1110 i = info.index - VFIO_PCI_NUM_REGIONS;
1111
1112 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
1113 info.size = vdev->region[i].size;
1114 info.flags = vdev->region[i].flags;
1115
1116 cap_type.type = vdev->region[i].type;
1117 cap_type.subtype = vdev->region[i].subtype;
1118
1119 ret = vfio_info_add_capability(&caps, &cap_type.header,
1120 sizeof(cap_type));
1121 if (ret)
1122 return ret;
1123
1124 if (vdev->region[i].ops->add_capability) {
1125 ret = vdev->region[i].ops->add_capability(
1126 vdev, &vdev->region[i], &caps);
1127 if (ret)
1128 return ret;
1129 }
1130 }
1131 }
1132
1133 if (caps.size) {
1134 info.flags |= VFIO_REGION_INFO_FLAG_CAPS;
1135 if (info.argsz < sizeof(info) + caps.size) {
1136 info.argsz = sizeof(info) + caps.size;
1137 info.cap_offset = 0;
1138 } else {
1139 vfio_info_cap_shift(&caps, sizeof(info));
1140 if (copy_to_user(arg + 1, caps.buf, caps.size)) {
1141 kfree(caps.buf);
1142 return -EFAULT;
1143 }
1144 info.cap_offset = sizeof(*arg);
1145 }
1146
1147 kfree(caps.buf);
1148 }
1149
1150 return copy_to_user(arg, &info, minsz) ? -EFAULT : 0;
1151 }
1152
vfio_pci_ioctl_get_irq_info(struct vfio_pci_core_device * vdev,struct vfio_irq_info __user * arg)1153 static int vfio_pci_ioctl_get_irq_info(struct vfio_pci_core_device *vdev,
1154 struct vfio_irq_info __user *arg)
1155 {
1156 unsigned long minsz = offsetofend(struct vfio_irq_info, count);
1157 struct vfio_irq_info info;
1158
1159 if (copy_from_user(&info, arg, minsz))
1160 return -EFAULT;
1161
1162 if (info.argsz < minsz || info.index >= VFIO_PCI_NUM_IRQS)
1163 return -EINVAL;
1164
1165 switch (info.index) {
1166 case VFIO_PCI_INTX_IRQ_INDEX ... VFIO_PCI_MSIX_IRQ_INDEX:
1167 case VFIO_PCI_REQ_IRQ_INDEX:
1168 break;
1169 case VFIO_PCI_ERR_IRQ_INDEX:
1170 if (pci_is_pcie(vdev->pdev))
1171 break;
1172 fallthrough;
1173 default:
1174 return -EINVAL;
1175 }
1176
1177 info.flags = VFIO_IRQ_INFO_EVENTFD;
1178
1179 info.count = vfio_pci_get_irq_count(vdev, info.index);
1180
1181 if (info.index == VFIO_PCI_INTX_IRQ_INDEX)
1182 info.flags |=
1183 (VFIO_IRQ_INFO_MASKABLE | VFIO_IRQ_INFO_AUTOMASKED);
1184 else if (info.index != VFIO_PCI_MSIX_IRQ_INDEX || !vdev->has_dyn_msix)
1185 info.flags |= VFIO_IRQ_INFO_NORESIZE;
1186
1187 return copy_to_user(arg, &info, minsz) ? -EFAULT : 0;
1188 }
1189
vfio_pci_ioctl_set_irqs(struct vfio_pci_core_device * vdev,struct vfio_irq_set __user * arg)1190 static int vfio_pci_ioctl_set_irqs(struct vfio_pci_core_device *vdev,
1191 struct vfio_irq_set __user *arg)
1192 {
1193 unsigned long minsz = offsetofend(struct vfio_irq_set, count);
1194 struct vfio_irq_set hdr;
1195 u8 *data = NULL;
1196 int max, ret = 0;
1197 size_t data_size = 0;
1198
1199 if (copy_from_user(&hdr, arg, minsz))
1200 return -EFAULT;
1201
1202 max = vfio_pci_get_irq_count(vdev, hdr.index);
1203
1204 ret = vfio_set_irqs_validate_and_prepare(&hdr, max, VFIO_PCI_NUM_IRQS,
1205 &data_size);
1206 if (ret)
1207 return ret;
1208
1209 if (data_size) {
1210 data = memdup_user(&arg->data, data_size);
1211 if (IS_ERR(data))
1212 return PTR_ERR(data);
1213 }
1214
1215 mutex_lock(&vdev->igate);
1216
1217 ret = vfio_pci_set_irqs_ioctl(vdev, hdr.flags, hdr.index, hdr.start,
1218 hdr.count, data);
1219
1220 mutex_unlock(&vdev->igate);
1221 kfree(data);
1222
1223 return ret;
1224 }
1225
vfio_pci_ioctl_reset(struct vfio_pci_core_device * vdev,void __user * arg)1226 static int vfio_pci_ioctl_reset(struct vfio_pci_core_device *vdev,
1227 void __user *arg)
1228 {
1229 int ret;
1230
1231 if (!vdev->reset_works)
1232 return -EINVAL;
1233
1234 vfio_pci_zap_and_down_write_memory_lock(vdev);
1235
1236 /*
1237 * This function can be invoked while the power state is non-D0. If
1238 * pci_try_reset_function() has been called while the power state is
1239 * non-D0, then pci_try_reset_function() will internally set the power
1240 * state to D0 without vfio driver involvement. For the devices which
1241 * have NoSoftRst-, the reset function can cause the PCI config space
1242 * reset without restoring the original state (saved locally in
1243 * 'vdev->pm_save').
1244 */
1245 vfio_pci_set_power_state(vdev, PCI_D0);
1246
1247 ret = pci_try_reset_function(vdev->pdev);
1248 up_write(&vdev->memory_lock);
1249
1250 return ret;
1251 }
1252
vfio_pci_ioctl_get_pci_hot_reset_info(struct vfio_pci_core_device * vdev,struct vfio_pci_hot_reset_info __user * arg)1253 static int vfio_pci_ioctl_get_pci_hot_reset_info(
1254 struct vfio_pci_core_device *vdev,
1255 struct vfio_pci_hot_reset_info __user *arg)
1256 {
1257 unsigned long minsz =
1258 offsetofend(struct vfio_pci_hot_reset_info, count);
1259 struct vfio_pci_dependent_device *devices = NULL;
1260 struct vfio_pci_hot_reset_info hdr;
1261 struct vfio_pci_fill_info fill = {};
1262 bool slot = false;
1263 int ret, count = 0;
1264
1265 if (copy_from_user(&hdr, arg, minsz))
1266 return -EFAULT;
1267
1268 if (hdr.argsz < minsz)
1269 return -EINVAL;
1270
1271 hdr.flags = 0;
1272
1273 /* Can we do a slot or bus reset or neither? */
1274 if (!pci_probe_reset_slot(vdev->pdev->slot))
1275 slot = true;
1276 else if (pci_probe_reset_bus(vdev->pdev->bus))
1277 return -ENODEV;
1278
1279 ret = vfio_pci_for_each_slot_or_bus(vdev->pdev, vfio_pci_count_devs,
1280 &count, slot);
1281 if (ret)
1282 return ret;
1283
1284 if (count > (hdr.argsz - sizeof(hdr)) / sizeof(*devices)) {
1285 hdr.count = count;
1286 ret = -ENOSPC;
1287 goto header;
1288 }
1289
1290 devices = kcalloc(count, sizeof(*devices), GFP_KERNEL);
1291 if (!devices)
1292 return -ENOMEM;
1293
1294 fill.devices = devices;
1295 fill.nr_devices = count;
1296 fill.vdev = &vdev->vdev;
1297
1298 if (vfio_device_cdev_opened(&vdev->vdev))
1299 fill.flags |= VFIO_PCI_HOT_RESET_FLAG_DEV_ID |
1300 VFIO_PCI_HOT_RESET_FLAG_DEV_ID_OWNED;
1301
1302 mutex_lock(&vdev->vdev.dev_set->lock);
1303 ret = vfio_pci_for_each_slot_or_bus(vdev->pdev, vfio_pci_fill_devs,
1304 &fill, slot);
1305 mutex_unlock(&vdev->vdev.dev_set->lock);
1306 if (ret)
1307 goto out;
1308
1309 if (copy_to_user(arg->devices, devices,
1310 sizeof(*devices) * fill.count)) {
1311 ret = -EFAULT;
1312 goto out;
1313 }
1314
1315 hdr.count = fill.count;
1316 hdr.flags = fill.flags;
1317
1318 header:
1319 if (copy_to_user(arg, &hdr, minsz))
1320 ret = -EFAULT;
1321 out:
1322 kfree(devices);
1323 return ret;
1324 }
1325
1326 static int
vfio_pci_ioctl_pci_hot_reset_groups(struct vfio_pci_core_device * vdev,int array_count,bool slot,struct vfio_pci_hot_reset __user * arg)1327 vfio_pci_ioctl_pci_hot_reset_groups(struct vfio_pci_core_device *vdev,
1328 int array_count, bool slot,
1329 struct vfio_pci_hot_reset __user *arg)
1330 {
1331 int32_t *group_fds;
1332 struct file **files;
1333 struct vfio_pci_group_info info;
1334 int file_idx, count = 0, ret = 0;
1335
1336 /*
1337 * We can't let userspace give us an arbitrarily large buffer to copy,
1338 * so verify how many we think there could be. Note groups can have
1339 * multiple devices so one group per device is the max.
1340 */
1341 ret = vfio_pci_for_each_slot_or_bus(vdev->pdev, vfio_pci_count_devs,
1342 &count, slot);
1343 if (ret)
1344 return ret;
1345
1346 if (array_count > count)
1347 return -EINVAL;
1348
1349 group_fds = kcalloc(array_count, sizeof(*group_fds), GFP_KERNEL);
1350 files = kcalloc(array_count, sizeof(*files), GFP_KERNEL);
1351 if (!group_fds || !files) {
1352 kfree(group_fds);
1353 kfree(files);
1354 return -ENOMEM;
1355 }
1356
1357 if (copy_from_user(group_fds, arg->group_fds,
1358 array_count * sizeof(*group_fds))) {
1359 kfree(group_fds);
1360 kfree(files);
1361 return -EFAULT;
1362 }
1363
1364 /*
1365 * Get the group file for each fd to ensure the group is held across
1366 * the reset
1367 */
1368 for (file_idx = 0; file_idx < array_count; file_idx++) {
1369 struct file *file = fget(group_fds[file_idx]);
1370
1371 if (!file) {
1372 ret = -EBADF;
1373 break;
1374 }
1375
1376 /* Ensure the FD is a vfio group FD.*/
1377 if (!vfio_file_is_group(file)) {
1378 fput(file);
1379 ret = -EINVAL;
1380 break;
1381 }
1382
1383 files[file_idx] = file;
1384 }
1385
1386 kfree(group_fds);
1387
1388 /* release reference to groups on error */
1389 if (ret)
1390 goto hot_reset_release;
1391
1392 info.count = array_count;
1393 info.files = files;
1394
1395 ret = vfio_pci_dev_set_hot_reset(vdev->vdev.dev_set, &info, NULL);
1396
1397 hot_reset_release:
1398 for (file_idx--; file_idx >= 0; file_idx--)
1399 fput(files[file_idx]);
1400
1401 kfree(files);
1402 return ret;
1403 }
1404
vfio_pci_ioctl_pci_hot_reset(struct vfio_pci_core_device * vdev,struct vfio_pci_hot_reset __user * arg)1405 static int vfio_pci_ioctl_pci_hot_reset(struct vfio_pci_core_device *vdev,
1406 struct vfio_pci_hot_reset __user *arg)
1407 {
1408 unsigned long minsz = offsetofend(struct vfio_pci_hot_reset, count);
1409 struct vfio_pci_hot_reset hdr;
1410 bool slot = false;
1411
1412 if (copy_from_user(&hdr, arg, minsz))
1413 return -EFAULT;
1414
1415 if (hdr.argsz < minsz || hdr.flags)
1416 return -EINVAL;
1417
1418 /* zero-length array is only for cdev opened devices */
1419 if (!!hdr.count == vfio_device_cdev_opened(&vdev->vdev))
1420 return -EINVAL;
1421
1422 /* Can we do a slot or bus reset or neither? */
1423 if (!pci_probe_reset_slot(vdev->pdev->slot))
1424 slot = true;
1425 else if (pci_probe_reset_bus(vdev->pdev->bus))
1426 return -ENODEV;
1427
1428 if (hdr.count)
1429 return vfio_pci_ioctl_pci_hot_reset_groups(vdev, hdr.count, slot, arg);
1430
1431 return vfio_pci_dev_set_hot_reset(vdev->vdev.dev_set, NULL,
1432 vfio_iommufd_device_ictx(&vdev->vdev));
1433 }
1434
vfio_pci_ioctl_ioeventfd(struct vfio_pci_core_device * vdev,struct vfio_device_ioeventfd __user * arg)1435 static int vfio_pci_ioctl_ioeventfd(struct vfio_pci_core_device *vdev,
1436 struct vfio_device_ioeventfd __user *arg)
1437 {
1438 unsigned long minsz = offsetofend(struct vfio_device_ioeventfd, fd);
1439 struct vfio_device_ioeventfd ioeventfd;
1440 int count;
1441
1442 if (copy_from_user(&ioeventfd, arg, minsz))
1443 return -EFAULT;
1444
1445 if (ioeventfd.argsz < minsz)
1446 return -EINVAL;
1447
1448 if (ioeventfd.flags & ~VFIO_DEVICE_IOEVENTFD_SIZE_MASK)
1449 return -EINVAL;
1450
1451 count = ioeventfd.flags & VFIO_DEVICE_IOEVENTFD_SIZE_MASK;
1452
1453 if (hweight8(count) != 1 || ioeventfd.fd < -1)
1454 return -EINVAL;
1455
1456 return vfio_pci_ioeventfd(vdev, ioeventfd.offset, ioeventfd.data, count,
1457 ioeventfd.fd);
1458 }
1459
vfio_pci_core_ioctl(struct vfio_device * core_vdev,unsigned int cmd,unsigned long arg)1460 long vfio_pci_core_ioctl(struct vfio_device *core_vdev, unsigned int cmd,
1461 unsigned long arg)
1462 {
1463 struct vfio_pci_core_device *vdev =
1464 container_of(core_vdev, struct vfio_pci_core_device, vdev);
1465 void __user *uarg = (void __user *)arg;
1466
1467 switch (cmd) {
1468 case VFIO_DEVICE_GET_INFO:
1469 return vfio_pci_ioctl_get_info(vdev, uarg);
1470 case VFIO_DEVICE_GET_IRQ_INFO:
1471 return vfio_pci_ioctl_get_irq_info(vdev, uarg);
1472 case VFIO_DEVICE_GET_PCI_HOT_RESET_INFO:
1473 return vfio_pci_ioctl_get_pci_hot_reset_info(vdev, uarg);
1474 case VFIO_DEVICE_GET_REGION_INFO:
1475 return vfio_pci_ioctl_get_region_info(vdev, uarg);
1476 case VFIO_DEVICE_IOEVENTFD:
1477 return vfio_pci_ioctl_ioeventfd(vdev, uarg);
1478 case VFIO_DEVICE_PCI_HOT_RESET:
1479 return vfio_pci_ioctl_pci_hot_reset(vdev, uarg);
1480 case VFIO_DEVICE_RESET:
1481 return vfio_pci_ioctl_reset(vdev, uarg);
1482 case VFIO_DEVICE_SET_IRQS:
1483 return vfio_pci_ioctl_set_irqs(vdev, uarg);
1484 default:
1485 return -ENOTTY;
1486 }
1487 }
1488 EXPORT_SYMBOL_GPL(vfio_pci_core_ioctl);
1489
vfio_pci_core_feature_token(struct vfio_device * device,u32 flags,uuid_t __user * arg,size_t argsz)1490 static int vfio_pci_core_feature_token(struct vfio_device *device, u32 flags,
1491 uuid_t __user *arg, size_t argsz)
1492 {
1493 struct vfio_pci_core_device *vdev =
1494 container_of(device, struct vfio_pci_core_device, vdev);
1495 uuid_t uuid;
1496 int ret;
1497
1498 if (!vdev->vf_token)
1499 return -ENOTTY;
1500 /*
1501 * We do not support GET of the VF Token UUID as this could
1502 * expose the token of the previous device user.
1503 */
1504 ret = vfio_check_feature(flags, argsz, VFIO_DEVICE_FEATURE_SET,
1505 sizeof(uuid));
1506 if (ret != 1)
1507 return ret;
1508
1509 if (copy_from_user(&uuid, arg, sizeof(uuid)))
1510 return -EFAULT;
1511
1512 mutex_lock(&vdev->vf_token->lock);
1513 uuid_copy(&vdev->vf_token->uuid, &uuid);
1514 mutex_unlock(&vdev->vf_token->lock);
1515 return 0;
1516 }
1517
vfio_pci_core_ioctl_feature(struct vfio_device * device,u32 flags,void __user * arg,size_t argsz)1518 int vfio_pci_core_ioctl_feature(struct vfio_device *device, u32 flags,
1519 void __user *arg, size_t argsz)
1520 {
1521 switch (flags & VFIO_DEVICE_FEATURE_MASK) {
1522 case VFIO_DEVICE_FEATURE_LOW_POWER_ENTRY:
1523 return vfio_pci_core_pm_entry(device, flags, arg, argsz);
1524 case VFIO_DEVICE_FEATURE_LOW_POWER_ENTRY_WITH_WAKEUP:
1525 return vfio_pci_core_pm_entry_with_wakeup(device, flags,
1526 arg, argsz);
1527 case VFIO_DEVICE_FEATURE_LOW_POWER_EXIT:
1528 return vfio_pci_core_pm_exit(device, flags, arg, argsz);
1529 case VFIO_DEVICE_FEATURE_PCI_VF_TOKEN:
1530 return vfio_pci_core_feature_token(device, flags, arg, argsz);
1531 default:
1532 return -ENOTTY;
1533 }
1534 }
1535 EXPORT_SYMBOL_GPL(vfio_pci_core_ioctl_feature);
1536
vfio_pci_rw(struct vfio_pci_core_device * vdev,char __user * buf,size_t count,loff_t * ppos,bool iswrite)1537 static ssize_t vfio_pci_rw(struct vfio_pci_core_device *vdev, char __user *buf,
1538 size_t count, loff_t *ppos, bool iswrite)
1539 {
1540 unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos);
1541 int ret;
1542
1543 if (index >= VFIO_PCI_NUM_REGIONS + vdev->num_regions)
1544 return -EINVAL;
1545
1546 ret = pm_runtime_resume_and_get(&vdev->pdev->dev);
1547 if (ret) {
1548 pci_info_ratelimited(vdev->pdev, "runtime resume failed %d\n",
1549 ret);
1550 return -EIO;
1551 }
1552
1553 switch (index) {
1554 case VFIO_PCI_CONFIG_REGION_INDEX:
1555 ret = vfio_pci_config_rw(vdev, buf, count, ppos, iswrite);
1556 break;
1557
1558 case VFIO_PCI_ROM_REGION_INDEX:
1559 if (iswrite)
1560 ret = -EINVAL;
1561 else
1562 ret = vfio_pci_bar_rw(vdev, buf, count, ppos, false);
1563 break;
1564
1565 case VFIO_PCI_BAR0_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX:
1566 ret = vfio_pci_bar_rw(vdev, buf, count, ppos, iswrite);
1567 break;
1568
1569 case VFIO_PCI_VGA_REGION_INDEX:
1570 ret = vfio_pci_vga_rw(vdev, buf, count, ppos, iswrite);
1571 break;
1572
1573 default:
1574 index -= VFIO_PCI_NUM_REGIONS;
1575 ret = vdev->region[index].ops->rw(vdev, buf,
1576 count, ppos, iswrite);
1577 break;
1578 }
1579
1580 pm_runtime_put(&vdev->pdev->dev);
1581 return ret;
1582 }
1583
vfio_pci_core_read(struct vfio_device * core_vdev,char __user * buf,size_t count,loff_t * ppos)1584 ssize_t vfio_pci_core_read(struct vfio_device *core_vdev, char __user *buf,
1585 size_t count, loff_t *ppos)
1586 {
1587 struct vfio_pci_core_device *vdev =
1588 container_of(core_vdev, struct vfio_pci_core_device, vdev);
1589
1590 if (!count)
1591 return 0;
1592
1593 return vfio_pci_rw(vdev, buf, count, ppos, false);
1594 }
1595 EXPORT_SYMBOL_GPL(vfio_pci_core_read);
1596
vfio_pci_core_write(struct vfio_device * core_vdev,const char __user * buf,size_t count,loff_t * ppos)1597 ssize_t vfio_pci_core_write(struct vfio_device *core_vdev, const char __user *buf,
1598 size_t count, loff_t *ppos)
1599 {
1600 struct vfio_pci_core_device *vdev =
1601 container_of(core_vdev, struct vfio_pci_core_device, vdev);
1602
1603 if (!count)
1604 return 0;
1605
1606 return vfio_pci_rw(vdev, (char __user *)buf, count, ppos, true);
1607 }
1608 EXPORT_SYMBOL_GPL(vfio_pci_core_write);
1609
1610 /* Return 1 on zap and vma_lock acquired, 0 on contention (only with @try) */
vfio_pci_zap_and_vma_lock(struct vfio_pci_core_device * vdev,bool try)1611 static int vfio_pci_zap_and_vma_lock(struct vfio_pci_core_device *vdev, bool try)
1612 {
1613 struct vfio_pci_mmap_vma *mmap_vma, *tmp;
1614
1615 /*
1616 * Lock ordering:
1617 * vma_lock is nested under mmap_lock for vm_ops callback paths.
1618 * The memory_lock semaphore is used by both code paths calling
1619 * into this function to zap vmas and the vm_ops.fault callback
1620 * to protect the memory enable state of the device.
1621 *
1622 * When zapping vmas we need to maintain the mmap_lock => vma_lock
1623 * ordering, which requires using vma_lock to walk vma_list to
1624 * acquire an mm, then dropping vma_lock to get the mmap_lock and
1625 * reacquiring vma_lock. This logic is derived from similar
1626 * requirements in uverbs_user_mmap_disassociate().
1627 *
1628 * mmap_lock must always be the top-level lock when it is taken.
1629 * Therefore we can only hold the memory_lock write lock when
1630 * vma_list is empty, as we'd need to take mmap_lock to clear
1631 * entries. vma_list can only be guaranteed empty when holding
1632 * vma_lock, thus memory_lock is nested under vma_lock.
1633 *
1634 * This enables the vm_ops.fault callback to acquire vma_lock,
1635 * followed by memory_lock read lock, while already holding
1636 * mmap_lock without risk of deadlock.
1637 */
1638 while (1) {
1639 struct mm_struct *mm = NULL;
1640
1641 if (try) {
1642 if (!mutex_trylock(&vdev->vma_lock))
1643 return 0;
1644 } else {
1645 mutex_lock(&vdev->vma_lock);
1646 }
1647 while (!list_empty(&vdev->vma_list)) {
1648 mmap_vma = list_first_entry(&vdev->vma_list,
1649 struct vfio_pci_mmap_vma,
1650 vma_next);
1651 mm = mmap_vma->vma->vm_mm;
1652 if (mmget_not_zero(mm))
1653 break;
1654
1655 list_del(&mmap_vma->vma_next);
1656 kfree(mmap_vma);
1657 mm = NULL;
1658 }
1659 if (!mm)
1660 return 1;
1661 mutex_unlock(&vdev->vma_lock);
1662
1663 if (try) {
1664 if (!mmap_read_trylock(mm)) {
1665 mmput(mm);
1666 return 0;
1667 }
1668 } else {
1669 mmap_read_lock(mm);
1670 }
1671 if (try) {
1672 if (!mutex_trylock(&vdev->vma_lock)) {
1673 mmap_read_unlock(mm);
1674 mmput(mm);
1675 return 0;
1676 }
1677 } else {
1678 mutex_lock(&vdev->vma_lock);
1679 }
1680 list_for_each_entry_safe(mmap_vma, tmp,
1681 &vdev->vma_list, vma_next) {
1682 struct vm_area_struct *vma = mmap_vma->vma;
1683
1684 if (vma->vm_mm != mm)
1685 continue;
1686
1687 list_del(&mmap_vma->vma_next);
1688 kfree(mmap_vma);
1689
1690 zap_vma_ptes(vma, vma->vm_start,
1691 vma->vm_end - vma->vm_start);
1692 }
1693 mutex_unlock(&vdev->vma_lock);
1694 mmap_read_unlock(mm);
1695 mmput(mm);
1696 }
1697 }
1698
vfio_pci_zap_and_down_write_memory_lock(struct vfio_pci_core_device * vdev)1699 void vfio_pci_zap_and_down_write_memory_lock(struct vfio_pci_core_device *vdev)
1700 {
1701 vfio_pci_zap_and_vma_lock(vdev, false);
1702 down_write(&vdev->memory_lock);
1703 mutex_unlock(&vdev->vma_lock);
1704 }
1705
vfio_pci_memory_lock_and_enable(struct vfio_pci_core_device * vdev)1706 u16 vfio_pci_memory_lock_and_enable(struct vfio_pci_core_device *vdev)
1707 {
1708 u16 cmd;
1709
1710 down_write(&vdev->memory_lock);
1711 pci_read_config_word(vdev->pdev, PCI_COMMAND, &cmd);
1712 if (!(cmd & PCI_COMMAND_MEMORY))
1713 pci_write_config_word(vdev->pdev, PCI_COMMAND,
1714 cmd | PCI_COMMAND_MEMORY);
1715
1716 return cmd;
1717 }
1718
vfio_pci_memory_unlock_and_restore(struct vfio_pci_core_device * vdev,u16 cmd)1719 void vfio_pci_memory_unlock_and_restore(struct vfio_pci_core_device *vdev, u16 cmd)
1720 {
1721 pci_write_config_word(vdev->pdev, PCI_COMMAND, cmd);
1722 up_write(&vdev->memory_lock);
1723 }
1724
1725 /* Caller holds vma_lock */
__vfio_pci_add_vma(struct vfio_pci_core_device * vdev,struct vm_area_struct * vma)1726 static int __vfio_pci_add_vma(struct vfio_pci_core_device *vdev,
1727 struct vm_area_struct *vma)
1728 {
1729 struct vfio_pci_mmap_vma *mmap_vma;
1730
1731 mmap_vma = kmalloc(sizeof(*mmap_vma), GFP_KERNEL_ACCOUNT);
1732 if (!mmap_vma)
1733 return -ENOMEM;
1734
1735 mmap_vma->vma = vma;
1736 list_add(&mmap_vma->vma_next, &vdev->vma_list);
1737
1738 return 0;
1739 }
1740
1741 /*
1742 * Zap mmaps on open so that we can fault them in on access and therefore
1743 * our vma_list only tracks mappings accessed since last zap.
1744 */
vfio_pci_mmap_open(struct vm_area_struct * vma)1745 static void vfio_pci_mmap_open(struct vm_area_struct *vma)
1746 {
1747 zap_vma_ptes(vma, vma->vm_start, vma->vm_end - vma->vm_start);
1748 }
1749
vfio_pci_mmap_close(struct vm_area_struct * vma)1750 static void vfio_pci_mmap_close(struct vm_area_struct *vma)
1751 {
1752 struct vfio_pci_core_device *vdev = vma->vm_private_data;
1753 struct vfio_pci_mmap_vma *mmap_vma;
1754
1755 mutex_lock(&vdev->vma_lock);
1756 list_for_each_entry(mmap_vma, &vdev->vma_list, vma_next) {
1757 if (mmap_vma->vma == vma) {
1758 list_del(&mmap_vma->vma_next);
1759 kfree(mmap_vma);
1760 break;
1761 }
1762 }
1763 mutex_unlock(&vdev->vma_lock);
1764 }
1765
vfio_pci_mmap_fault(struct vm_fault * vmf)1766 static vm_fault_t vfio_pci_mmap_fault(struct vm_fault *vmf)
1767 {
1768 struct vm_area_struct *vma = vmf->vma;
1769 struct vfio_pci_core_device *vdev = vma->vm_private_data;
1770 struct vfio_pci_mmap_vma *mmap_vma;
1771 vm_fault_t ret = VM_FAULT_NOPAGE;
1772
1773 mutex_lock(&vdev->vma_lock);
1774 down_read(&vdev->memory_lock);
1775
1776 /*
1777 * Memory region cannot be accessed if the low power feature is engaged
1778 * or memory access is disabled.
1779 */
1780 if (vdev->pm_runtime_engaged || !__vfio_pci_memory_enabled(vdev)) {
1781 ret = VM_FAULT_SIGBUS;
1782 goto up_out;
1783 }
1784
1785 /*
1786 * We populate the whole vma on fault, so we need to test whether
1787 * the vma has already been mapped, such as for concurrent faults
1788 * to the same vma. io_remap_pfn_range() will trigger a BUG_ON if
1789 * we ask it to fill the same range again.
1790 */
1791 list_for_each_entry(mmap_vma, &vdev->vma_list, vma_next) {
1792 if (mmap_vma->vma == vma)
1793 goto up_out;
1794 }
1795
1796 if (io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
1797 vma->vm_end - vma->vm_start,
1798 vma->vm_page_prot)) {
1799 ret = VM_FAULT_SIGBUS;
1800 zap_vma_ptes(vma, vma->vm_start, vma->vm_end - vma->vm_start);
1801 goto up_out;
1802 }
1803
1804 if (__vfio_pci_add_vma(vdev, vma)) {
1805 ret = VM_FAULT_OOM;
1806 zap_vma_ptes(vma, vma->vm_start, vma->vm_end - vma->vm_start);
1807 }
1808
1809 up_out:
1810 up_read(&vdev->memory_lock);
1811 mutex_unlock(&vdev->vma_lock);
1812 return ret;
1813 }
1814
1815 static const struct vm_operations_struct vfio_pci_mmap_ops = {
1816 .open = vfio_pci_mmap_open,
1817 .close = vfio_pci_mmap_close,
1818 .fault = vfio_pci_mmap_fault,
1819 };
1820
vfio_pci_core_mmap(struct vfio_device * core_vdev,struct vm_area_struct * vma)1821 int vfio_pci_core_mmap(struct vfio_device *core_vdev, struct vm_area_struct *vma)
1822 {
1823 struct vfio_pci_core_device *vdev =
1824 container_of(core_vdev, struct vfio_pci_core_device, vdev);
1825 struct pci_dev *pdev = vdev->pdev;
1826 unsigned int index;
1827 u64 phys_len, req_len, pgoff, req_start;
1828 int ret;
1829
1830 index = vma->vm_pgoff >> (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT);
1831
1832 if (index >= VFIO_PCI_NUM_REGIONS + vdev->num_regions)
1833 return -EINVAL;
1834 if (vma->vm_end < vma->vm_start)
1835 return -EINVAL;
1836 if ((vma->vm_flags & VM_SHARED) == 0)
1837 return -EINVAL;
1838 if (index >= VFIO_PCI_NUM_REGIONS) {
1839 int regnum = index - VFIO_PCI_NUM_REGIONS;
1840 struct vfio_pci_region *region = vdev->region + regnum;
1841
1842 if (region->ops && region->ops->mmap &&
1843 (region->flags & VFIO_REGION_INFO_FLAG_MMAP))
1844 return region->ops->mmap(vdev, region, vma);
1845 return -EINVAL;
1846 }
1847 if (index >= VFIO_PCI_ROM_REGION_INDEX)
1848 return -EINVAL;
1849 if (!vdev->bar_mmap_supported[index])
1850 return -EINVAL;
1851
1852 phys_len = PAGE_ALIGN(pci_resource_len(pdev, index));
1853 req_len = vma->vm_end - vma->vm_start;
1854 pgoff = vma->vm_pgoff &
1855 ((1U << (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT)) - 1);
1856 req_start = pgoff << PAGE_SHIFT;
1857
1858 if (req_start + req_len > phys_len)
1859 return -EINVAL;
1860
1861 /*
1862 * Even though we don't make use of the barmap for the mmap,
1863 * we need to request the region and the barmap tracks that.
1864 */
1865 if (!vdev->barmap[index]) {
1866 ret = pci_request_selected_regions(pdev,
1867 1 << index, "vfio-pci");
1868 if (ret)
1869 return ret;
1870
1871 vdev->barmap[index] = pci_iomap(pdev, index, 0);
1872 if (!vdev->barmap[index]) {
1873 pci_release_selected_regions(pdev, 1 << index);
1874 return -ENOMEM;
1875 }
1876 }
1877
1878 vma->vm_private_data = vdev;
1879 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
1880 vma->vm_pgoff = (pci_resource_start(pdev, index) >> PAGE_SHIFT) + pgoff;
1881
1882 /*
1883 * See remap_pfn_range(), called from vfio_pci_fault() but we can't
1884 * change vm_flags within the fault handler. Set them now.
1885 */
1886 vm_flags_set(vma, VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP);
1887 vma->vm_ops = &vfio_pci_mmap_ops;
1888
1889 return 0;
1890 }
1891 EXPORT_SYMBOL_GPL(vfio_pci_core_mmap);
1892
vfio_pci_core_request(struct vfio_device * core_vdev,unsigned int count)1893 void vfio_pci_core_request(struct vfio_device *core_vdev, unsigned int count)
1894 {
1895 struct vfio_pci_core_device *vdev =
1896 container_of(core_vdev, struct vfio_pci_core_device, vdev);
1897 struct pci_dev *pdev = vdev->pdev;
1898
1899 mutex_lock(&vdev->igate);
1900
1901 if (vdev->req_trigger) {
1902 if (!(count % 10))
1903 pci_notice_ratelimited(pdev,
1904 "Relaying device request to user (#%u)\n",
1905 count);
1906 eventfd_signal(vdev->req_trigger, 1);
1907 } else if (count == 0) {
1908 pci_warn(pdev,
1909 "No device request channel registered, blocked until released by user\n");
1910 }
1911
1912 mutex_unlock(&vdev->igate);
1913 }
1914 EXPORT_SYMBOL_GPL(vfio_pci_core_request);
1915
vfio_pci_validate_vf_token(struct vfio_pci_core_device * vdev,bool vf_token,uuid_t * uuid)1916 static int vfio_pci_validate_vf_token(struct vfio_pci_core_device *vdev,
1917 bool vf_token, uuid_t *uuid)
1918 {
1919 /*
1920 * There's always some degree of trust or collaboration between SR-IOV
1921 * PF and VFs, even if just that the PF hosts the SR-IOV capability and
1922 * can disrupt VFs with a reset, but often the PF has more explicit
1923 * access to deny service to the VF or access data passed through the
1924 * VF. We therefore require an opt-in via a shared VF token (UUID) to
1925 * represent this trust. This both prevents that a VF driver might
1926 * assume the PF driver is a trusted, in-kernel driver, and also that
1927 * a PF driver might be replaced with a rogue driver, unknown to in-use
1928 * VF drivers.
1929 *
1930 * Therefore when presented with a VF, if the PF is a vfio device and
1931 * it is bound to the vfio-pci driver, the user needs to provide a VF
1932 * token to access the device, in the form of appending a vf_token to
1933 * the device name, for example:
1934 *
1935 * "0000:04:10.0 vf_token=bd8d9d2b-5a5f-4f5a-a211-f591514ba1f3"
1936 *
1937 * When presented with a PF which has VFs in use, the user must also
1938 * provide the current VF token to prove collaboration with existing
1939 * VF users. If VFs are not in use, the VF token provided for the PF
1940 * device will act to set the VF token.
1941 *
1942 * If the VF token is provided but unused, an error is generated.
1943 */
1944 if (vdev->pdev->is_virtfn) {
1945 struct vfio_pci_core_device *pf_vdev = vdev->sriov_pf_core_dev;
1946 bool match;
1947
1948 if (!pf_vdev) {
1949 if (!vf_token)
1950 return 0; /* PF is not vfio-pci, no VF token */
1951
1952 pci_info_ratelimited(vdev->pdev,
1953 "VF token incorrectly provided, PF not bound to vfio-pci\n");
1954 return -EINVAL;
1955 }
1956
1957 if (!vf_token) {
1958 pci_info_ratelimited(vdev->pdev,
1959 "VF token required to access device\n");
1960 return -EACCES;
1961 }
1962
1963 mutex_lock(&pf_vdev->vf_token->lock);
1964 match = uuid_equal(uuid, &pf_vdev->vf_token->uuid);
1965 mutex_unlock(&pf_vdev->vf_token->lock);
1966
1967 if (!match) {
1968 pci_info_ratelimited(vdev->pdev,
1969 "Incorrect VF token provided for device\n");
1970 return -EACCES;
1971 }
1972 } else if (vdev->vf_token) {
1973 mutex_lock(&vdev->vf_token->lock);
1974 if (vdev->vf_token->users) {
1975 if (!vf_token) {
1976 mutex_unlock(&vdev->vf_token->lock);
1977 pci_info_ratelimited(vdev->pdev,
1978 "VF token required to access device\n");
1979 return -EACCES;
1980 }
1981
1982 if (!uuid_equal(uuid, &vdev->vf_token->uuid)) {
1983 mutex_unlock(&vdev->vf_token->lock);
1984 pci_info_ratelimited(vdev->pdev,
1985 "Incorrect VF token provided for device\n");
1986 return -EACCES;
1987 }
1988 } else if (vf_token) {
1989 uuid_copy(&vdev->vf_token->uuid, uuid);
1990 }
1991
1992 mutex_unlock(&vdev->vf_token->lock);
1993 } else if (vf_token) {
1994 pci_info_ratelimited(vdev->pdev,
1995 "VF token incorrectly provided, not a PF or VF\n");
1996 return -EINVAL;
1997 }
1998
1999 return 0;
2000 }
2001
2002 #define VF_TOKEN_ARG "vf_token="
2003
vfio_pci_core_match(struct vfio_device * core_vdev,char * buf)2004 int vfio_pci_core_match(struct vfio_device *core_vdev, char *buf)
2005 {
2006 struct vfio_pci_core_device *vdev =
2007 container_of(core_vdev, struct vfio_pci_core_device, vdev);
2008 bool vf_token = false;
2009 uuid_t uuid;
2010 int ret;
2011
2012 if (strncmp(pci_name(vdev->pdev), buf, strlen(pci_name(vdev->pdev))))
2013 return 0; /* No match */
2014
2015 if (strlen(buf) > strlen(pci_name(vdev->pdev))) {
2016 buf += strlen(pci_name(vdev->pdev));
2017
2018 if (*buf != ' ')
2019 return 0; /* No match: non-whitespace after name */
2020
2021 while (*buf) {
2022 if (*buf == ' ') {
2023 buf++;
2024 continue;
2025 }
2026
2027 if (!vf_token && !strncmp(buf, VF_TOKEN_ARG,
2028 strlen(VF_TOKEN_ARG))) {
2029 buf += strlen(VF_TOKEN_ARG);
2030
2031 if (strlen(buf) < UUID_STRING_LEN)
2032 return -EINVAL;
2033
2034 ret = uuid_parse(buf, &uuid);
2035 if (ret)
2036 return ret;
2037
2038 vf_token = true;
2039 buf += UUID_STRING_LEN;
2040 } else {
2041 /* Unknown/duplicate option */
2042 return -EINVAL;
2043 }
2044 }
2045 }
2046
2047 ret = vfio_pci_validate_vf_token(vdev, vf_token, &uuid);
2048 if (ret)
2049 return ret;
2050
2051 return 1; /* Match */
2052 }
2053 EXPORT_SYMBOL_GPL(vfio_pci_core_match);
2054
vfio_pci_bus_notifier(struct notifier_block * nb,unsigned long action,void * data)2055 static int vfio_pci_bus_notifier(struct notifier_block *nb,
2056 unsigned long action, void *data)
2057 {
2058 struct vfio_pci_core_device *vdev = container_of(nb,
2059 struct vfio_pci_core_device, nb);
2060 struct device *dev = data;
2061 struct pci_dev *pdev = to_pci_dev(dev);
2062 struct pci_dev *physfn = pci_physfn(pdev);
2063
2064 if (action == BUS_NOTIFY_ADD_DEVICE &&
2065 pdev->is_virtfn && physfn == vdev->pdev) {
2066 pci_info(vdev->pdev, "Captured SR-IOV VF %s driver_override\n",
2067 pci_name(pdev));
2068 pdev->driver_override = kasprintf(GFP_KERNEL, "%s",
2069 vdev->vdev.ops->name);
2070 } else if (action == BUS_NOTIFY_BOUND_DRIVER &&
2071 pdev->is_virtfn && physfn == vdev->pdev) {
2072 struct pci_driver *drv = pci_dev_driver(pdev);
2073
2074 if (drv && drv != pci_dev_driver(vdev->pdev))
2075 pci_warn(vdev->pdev,
2076 "VF %s bound to driver %s while PF bound to driver %s\n",
2077 pci_name(pdev), drv->name,
2078 pci_dev_driver(vdev->pdev)->name);
2079 }
2080
2081 return 0;
2082 }
2083
vfio_pci_vf_init(struct vfio_pci_core_device * vdev)2084 static int vfio_pci_vf_init(struct vfio_pci_core_device *vdev)
2085 {
2086 struct pci_dev *pdev = vdev->pdev;
2087 struct vfio_pci_core_device *cur;
2088 struct pci_dev *physfn;
2089 int ret;
2090
2091 if (pdev->is_virtfn) {
2092 /*
2093 * If this VF was created by our vfio_pci_core_sriov_configure()
2094 * then we can find the PF vfio_pci_core_device now, and due to
2095 * the locking in pci_disable_sriov() it cannot change until
2096 * this VF device driver is removed.
2097 */
2098 physfn = pci_physfn(vdev->pdev);
2099 mutex_lock(&vfio_pci_sriov_pfs_mutex);
2100 list_for_each_entry(cur, &vfio_pci_sriov_pfs, sriov_pfs_item) {
2101 if (cur->pdev == physfn) {
2102 vdev->sriov_pf_core_dev = cur;
2103 break;
2104 }
2105 }
2106 mutex_unlock(&vfio_pci_sriov_pfs_mutex);
2107 return 0;
2108 }
2109
2110 /* Not a SRIOV PF */
2111 if (!pdev->is_physfn)
2112 return 0;
2113
2114 vdev->vf_token = kzalloc(sizeof(*vdev->vf_token), GFP_KERNEL);
2115 if (!vdev->vf_token)
2116 return -ENOMEM;
2117
2118 mutex_init(&vdev->vf_token->lock);
2119 uuid_gen(&vdev->vf_token->uuid);
2120
2121 vdev->nb.notifier_call = vfio_pci_bus_notifier;
2122 ret = bus_register_notifier(&pci_bus_type, &vdev->nb);
2123 if (ret) {
2124 kfree(vdev->vf_token);
2125 return ret;
2126 }
2127 return 0;
2128 }
2129
vfio_pci_vf_uninit(struct vfio_pci_core_device * vdev)2130 static void vfio_pci_vf_uninit(struct vfio_pci_core_device *vdev)
2131 {
2132 if (!vdev->vf_token)
2133 return;
2134
2135 bus_unregister_notifier(&pci_bus_type, &vdev->nb);
2136 WARN_ON(vdev->vf_token->users);
2137 mutex_destroy(&vdev->vf_token->lock);
2138 kfree(vdev->vf_token);
2139 }
2140
vfio_pci_vga_init(struct vfio_pci_core_device * vdev)2141 static int vfio_pci_vga_init(struct vfio_pci_core_device *vdev)
2142 {
2143 struct pci_dev *pdev = vdev->pdev;
2144 int ret;
2145
2146 if (!vfio_pci_is_vga(pdev))
2147 return 0;
2148
2149 ret = aperture_remove_conflicting_pci_devices(pdev, vdev->vdev.ops->name);
2150 if (ret)
2151 return ret;
2152
2153 ret = vga_client_register(pdev, vfio_pci_set_decode);
2154 if (ret)
2155 return ret;
2156 vga_set_legacy_decoding(pdev, vfio_pci_set_decode(pdev, false));
2157 return 0;
2158 }
2159
vfio_pci_vga_uninit(struct vfio_pci_core_device * vdev)2160 static void vfio_pci_vga_uninit(struct vfio_pci_core_device *vdev)
2161 {
2162 struct pci_dev *pdev = vdev->pdev;
2163
2164 if (!vfio_pci_is_vga(pdev))
2165 return;
2166 vga_client_unregister(pdev);
2167 vga_set_legacy_decoding(pdev, VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM |
2168 VGA_RSRC_LEGACY_IO |
2169 VGA_RSRC_LEGACY_MEM);
2170 }
2171
vfio_pci_core_init_dev(struct vfio_device * core_vdev)2172 int vfio_pci_core_init_dev(struct vfio_device *core_vdev)
2173 {
2174 struct vfio_pci_core_device *vdev =
2175 container_of(core_vdev, struct vfio_pci_core_device, vdev);
2176
2177 vdev->pdev = to_pci_dev(core_vdev->dev);
2178 vdev->irq_type = VFIO_PCI_NUM_IRQS;
2179 mutex_init(&vdev->igate);
2180 spin_lock_init(&vdev->irqlock);
2181 mutex_init(&vdev->ioeventfds_lock);
2182 INIT_LIST_HEAD(&vdev->dummy_resources_list);
2183 INIT_LIST_HEAD(&vdev->ioeventfds_list);
2184 mutex_init(&vdev->vma_lock);
2185 INIT_LIST_HEAD(&vdev->vma_list);
2186 INIT_LIST_HEAD(&vdev->sriov_pfs_item);
2187 init_rwsem(&vdev->memory_lock);
2188 xa_init(&vdev->ctx);
2189
2190 return 0;
2191 }
2192 EXPORT_SYMBOL_GPL(vfio_pci_core_init_dev);
2193
vfio_pci_core_release_dev(struct vfio_device * core_vdev)2194 void vfio_pci_core_release_dev(struct vfio_device *core_vdev)
2195 {
2196 struct vfio_pci_core_device *vdev =
2197 container_of(core_vdev, struct vfio_pci_core_device, vdev);
2198
2199 mutex_destroy(&vdev->igate);
2200 mutex_destroy(&vdev->ioeventfds_lock);
2201 mutex_destroy(&vdev->vma_lock);
2202 kfree(vdev->region);
2203 kfree(vdev->pm_save);
2204 }
2205 EXPORT_SYMBOL_GPL(vfio_pci_core_release_dev);
2206
vfio_pci_core_register_device(struct vfio_pci_core_device * vdev)2207 int vfio_pci_core_register_device(struct vfio_pci_core_device *vdev)
2208 {
2209 struct pci_dev *pdev = vdev->pdev;
2210 struct device *dev = &pdev->dev;
2211 int ret;
2212
2213 /* Drivers must set the vfio_pci_core_device to their drvdata */
2214 if (WARN_ON(vdev != dev_get_drvdata(dev)))
2215 return -EINVAL;
2216
2217 if (pdev->hdr_type != PCI_HEADER_TYPE_NORMAL)
2218 return -EINVAL;
2219
2220 if (vdev->vdev.mig_ops) {
2221 if (!(vdev->vdev.mig_ops->migration_get_state &&
2222 vdev->vdev.mig_ops->migration_set_state &&
2223 vdev->vdev.mig_ops->migration_get_data_size) ||
2224 !(vdev->vdev.migration_flags & VFIO_MIGRATION_STOP_COPY))
2225 return -EINVAL;
2226 }
2227
2228 if (vdev->vdev.log_ops && !(vdev->vdev.log_ops->log_start &&
2229 vdev->vdev.log_ops->log_stop &&
2230 vdev->vdev.log_ops->log_read_and_clear))
2231 return -EINVAL;
2232
2233 /*
2234 * Prevent binding to PFs with VFs enabled, the VFs might be in use
2235 * by the host or other users. We cannot capture the VFs if they
2236 * already exist, nor can we track VF users. Disabling SR-IOV here
2237 * would initiate removing the VFs, which would unbind the driver,
2238 * which is prone to blocking if that VF is also in use by vfio-pci.
2239 * Just reject these PFs and let the user sort it out.
2240 */
2241 if (pci_num_vf(pdev)) {
2242 pci_warn(pdev, "Cannot bind to PF with SR-IOV enabled\n");
2243 return -EBUSY;
2244 }
2245
2246 if (pci_is_root_bus(pdev->bus)) {
2247 ret = vfio_assign_device_set(&vdev->vdev, vdev);
2248 } else if (!pci_probe_reset_slot(pdev->slot)) {
2249 ret = vfio_assign_device_set(&vdev->vdev, pdev->slot);
2250 } else {
2251 /*
2252 * If there is no slot reset support for this device, the whole
2253 * bus needs to be grouped together to support bus-wide resets.
2254 */
2255 ret = vfio_assign_device_set(&vdev->vdev, pdev->bus);
2256 }
2257
2258 if (ret)
2259 return ret;
2260 ret = vfio_pci_vf_init(vdev);
2261 if (ret)
2262 return ret;
2263 ret = vfio_pci_vga_init(vdev);
2264 if (ret)
2265 goto out_vf;
2266
2267 vfio_pci_probe_power_state(vdev);
2268
2269 /*
2270 * pci-core sets the device power state to an unknown value at
2271 * bootup and after being removed from a driver. The only
2272 * transition it allows from this unknown state is to D0, which
2273 * typically happens when a driver calls pci_enable_device().
2274 * We're not ready to enable the device yet, but we do want to
2275 * be able to get to D3. Therefore first do a D0 transition
2276 * before enabling runtime PM.
2277 */
2278 vfio_pci_set_power_state(vdev, PCI_D0);
2279
2280 dev->driver->pm = &vfio_pci_core_pm_ops;
2281 pm_runtime_allow(dev);
2282 if (!disable_idle_d3)
2283 pm_runtime_put(dev);
2284
2285 ret = vfio_register_group_dev(&vdev->vdev);
2286 if (ret)
2287 goto out_power;
2288 return 0;
2289
2290 out_power:
2291 if (!disable_idle_d3)
2292 pm_runtime_get_noresume(dev);
2293
2294 pm_runtime_forbid(dev);
2295 out_vf:
2296 vfio_pci_vf_uninit(vdev);
2297 return ret;
2298 }
2299 EXPORT_SYMBOL_GPL(vfio_pci_core_register_device);
2300
vfio_pci_core_unregister_device(struct vfio_pci_core_device * vdev)2301 void vfio_pci_core_unregister_device(struct vfio_pci_core_device *vdev)
2302 {
2303 vfio_pci_core_sriov_configure(vdev, 0);
2304
2305 vfio_unregister_group_dev(&vdev->vdev);
2306
2307 vfio_pci_vf_uninit(vdev);
2308 vfio_pci_vga_uninit(vdev);
2309
2310 if (!disable_idle_d3)
2311 pm_runtime_get_noresume(&vdev->pdev->dev);
2312
2313 pm_runtime_forbid(&vdev->pdev->dev);
2314 }
2315 EXPORT_SYMBOL_GPL(vfio_pci_core_unregister_device);
2316
vfio_pci_core_aer_err_detected(struct pci_dev * pdev,pci_channel_state_t state)2317 pci_ers_result_t vfio_pci_core_aer_err_detected(struct pci_dev *pdev,
2318 pci_channel_state_t state)
2319 {
2320 struct vfio_pci_core_device *vdev = dev_get_drvdata(&pdev->dev);
2321
2322 mutex_lock(&vdev->igate);
2323
2324 if (vdev->err_trigger)
2325 eventfd_signal(vdev->err_trigger, 1);
2326
2327 mutex_unlock(&vdev->igate);
2328
2329 return PCI_ERS_RESULT_CAN_RECOVER;
2330 }
2331 EXPORT_SYMBOL_GPL(vfio_pci_core_aer_err_detected);
2332
vfio_pci_core_sriov_configure(struct vfio_pci_core_device * vdev,int nr_virtfn)2333 int vfio_pci_core_sriov_configure(struct vfio_pci_core_device *vdev,
2334 int nr_virtfn)
2335 {
2336 struct pci_dev *pdev = vdev->pdev;
2337 int ret = 0;
2338
2339 device_lock_assert(&pdev->dev);
2340
2341 if (nr_virtfn) {
2342 mutex_lock(&vfio_pci_sriov_pfs_mutex);
2343 /*
2344 * The thread that adds the vdev to the list is the only thread
2345 * that gets to call pci_enable_sriov() and we will only allow
2346 * it to be called once without going through
2347 * pci_disable_sriov()
2348 */
2349 if (!list_empty(&vdev->sriov_pfs_item)) {
2350 ret = -EINVAL;
2351 goto out_unlock;
2352 }
2353 list_add_tail(&vdev->sriov_pfs_item, &vfio_pci_sriov_pfs);
2354 mutex_unlock(&vfio_pci_sriov_pfs_mutex);
2355
2356 /*
2357 * The PF power state should always be higher than the VF power
2358 * state. The PF can be in low power state either with runtime
2359 * power management (when there is no user) or PCI_PM_CTRL
2360 * register write by the user. If PF is in the low power state,
2361 * then change the power state to D0 first before enabling
2362 * SR-IOV. Also, this function can be called at any time, and
2363 * userspace PCI_PM_CTRL write can race against this code path,
2364 * so protect the same with 'memory_lock'.
2365 */
2366 ret = pm_runtime_resume_and_get(&pdev->dev);
2367 if (ret)
2368 goto out_del;
2369
2370 down_write(&vdev->memory_lock);
2371 vfio_pci_set_power_state(vdev, PCI_D0);
2372 ret = pci_enable_sriov(pdev, nr_virtfn);
2373 up_write(&vdev->memory_lock);
2374 if (ret) {
2375 pm_runtime_put(&pdev->dev);
2376 goto out_del;
2377 }
2378 return nr_virtfn;
2379 }
2380
2381 if (pci_num_vf(pdev)) {
2382 pci_disable_sriov(pdev);
2383 pm_runtime_put(&pdev->dev);
2384 }
2385
2386 out_del:
2387 mutex_lock(&vfio_pci_sriov_pfs_mutex);
2388 list_del_init(&vdev->sriov_pfs_item);
2389 out_unlock:
2390 mutex_unlock(&vfio_pci_sriov_pfs_mutex);
2391 return ret;
2392 }
2393 EXPORT_SYMBOL_GPL(vfio_pci_core_sriov_configure);
2394
2395 const struct pci_error_handlers vfio_pci_core_err_handlers = {
2396 .error_detected = vfio_pci_core_aer_err_detected,
2397 };
2398 EXPORT_SYMBOL_GPL(vfio_pci_core_err_handlers);
2399
vfio_dev_in_groups(struct vfio_device * vdev,struct vfio_pci_group_info * groups)2400 static bool vfio_dev_in_groups(struct vfio_device *vdev,
2401 struct vfio_pci_group_info *groups)
2402 {
2403 unsigned int i;
2404
2405 if (!groups)
2406 return false;
2407
2408 for (i = 0; i < groups->count; i++)
2409 if (vfio_file_has_dev(groups->files[i], vdev))
2410 return true;
2411 return false;
2412 }
2413
vfio_pci_is_device_in_set(struct pci_dev * pdev,void * data)2414 static int vfio_pci_is_device_in_set(struct pci_dev *pdev, void *data)
2415 {
2416 struct vfio_device_set *dev_set = data;
2417
2418 return vfio_find_device_in_devset(dev_set, &pdev->dev) ? 0 : -ENODEV;
2419 }
2420
2421 /*
2422 * vfio-core considers a group to be viable and will create a vfio_device even
2423 * if some devices are bound to drivers like pci-stub or pcieport. Here we
2424 * require all PCI devices to be inside our dev_set since that ensures they stay
2425 * put and that every driver controlling the device can co-ordinate with the
2426 * device reset.
2427 *
2428 * Returns the pci_dev to pass to pci_reset_bus() if every PCI device to be
2429 * reset is inside the dev_set, and pci_reset_bus() can succeed. NULL otherwise.
2430 */
2431 static struct pci_dev *
vfio_pci_dev_set_resettable(struct vfio_device_set * dev_set)2432 vfio_pci_dev_set_resettable(struct vfio_device_set *dev_set)
2433 {
2434 struct pci_dev *pdev;
2435
2436 lockdep_assert_held(&dev_set->lock);
2437
2438 /*
2439 * By definition all PCI devices in the dev_set share the same PCI
2440 * reset, so any pci_dev will have the same outcomes for
2441 * pci_probe_reset_*() and pci_reset_bus().
2442 */
2443 pdev = list_first_entry(&dev_set->device_list,
2444 struct vfio_pci_core_device,
2445 vdev.dev_set_list)->pdev;
2446
2447 /* pci_reset_bus() is supported */
2448 if (pci_probe_reset_slot(pdev->slot) && pci_probe_reset_bus(pdev->bus))
2449 return NULL;
2450
2451 if (vfio_pci_for_each_slot_or_bus(pdev, vfio_pci_is_device_in_set,
2452 dev_set,
2453 !pci_probe_reset_slot(pdev->slot)))
2454 return NULL;
2455 return pdev;
2456 }
2457
vfio_pci_dev_set_pm_runtime_get(struct vfio_device_set * dev_set)2458 static int vfio_pci_dev_set_pm_runtime_get(struct vfio_device_set *dev_set)
2459 {
2460 struct vfio_pci_core_device *cur;
2461 int ret;
2462
2463 list_for_each_entry(cur, &dev_set->device_list, vdev.dev_set_list) {
2464 ret = pm_runtime_resume_and_get(&cur->pdev->dev);
2465 if (ret)
2466 goto unwind;
2467 }
2468
2469 return 0;
2470
2471 unwind:
2472 list_for_each_entry_continue_reverse(cur, &dev_set->device_list,
2473 vdev.dev_set_list)
2474 pm_runtime_put(&cur->pdev->dev);
2475
2476 return ret;
2477 }
2478
2479 /*
2480 * We need to get memory_lock for each device, but devices can share mmap_lock,
2481 * therefore we need to zap and hold the vma_lock for each device, and only then
2482 * get each memory_lock.
2483 */
vfio_pci_dev_set_hot_reset(struct vfio_device_set * dev_set,struct vfio_pci_group_info * groups,struct iommufd_ctx * iommufd_ctx)2484 static int vfio_pci_dev_set_hot_reset(struct vfio_device_set *dev_set,
2485 struct vfio_pci_group_info *groups,
2486 struct iommufd_ctx *iommufd_ctx)
2487 {
2488 struct vfio_pci_core_device *cur_mem;
2489 struct vfio_pci_core_device *cur_vma;
2490 struct vfio_pci_core_device *cur;
2491 struct pci_dev *pdev;
2492 bool is_mem = true;
2493 int ret;
2494
2495 mutex_lock(&dev_set->lock);
2496 cur_mem = list_first_entry(&dev_set->device_list,
2497 struct vfio_pci_core_device,
2498 vdev.dev_set_list);
2499
2500 pdev = vfio_pci_dev_set_resettable(dev_set);
2501 if (!pdev) {
2502 ret = -EINVAL;
2503 goto err_unlock;
2504 }
2505
2506 /*
2507 * Some of the devices in the dev_set can be in the runtime suspended
2508 * state. Increment the usage count for all the devices in the dev_set
2509 * before reset and decrement the same after reset.
2510 */
2511 ret = vfio_pci_dev_set_pm_runtime_get(dev_set);
2512 if (ret)
2513 goto err_unlock;
2514
2515 list_for_each_entry(cur_vma, &dev_set->device_list, vdev.dev_set_list) {
2516 bool owned;
2517
2518 /*
2519 * Test whether all the affected devices can be reset by the
2520 * user.
2521 *
2522 * If called from a group opened device and the user provides
2523 * a set of groups, all the devices in the dev_set should be
2524 * contained by the set of groups provided by the user.
2525 *
2526 * If called from a cdev opened device and the user provides
2527 * a zero-length array, all the devices in the dev_set must
2528 * be bound to the same iommufd_ctx as the input iommufd_ctx.
2529 * If there is any device that has not been bound to any
2530 * iommufd_ctx yet, check if its iommu_group has any device
2531 * bound to the input iommufd_ctx. Such devices can be
2532 * considered owned by the input iommufd_ctx as the device
2533 * cannot be owned by another iommufd_ctx when its iommu_group
2534 * is owned.
2535 *
2536 * Otherwise, reset is not allowed.
2537 */
2538 if (iommufd_ctx) {
2539 int devid = vfio_iommufd_get_dev_id(&cur_vma->vdev,
2540 iommufd_ctx);
2541
2542 owned = (devid > 0 || devid == -ENOENT);
2543 } else {
2544 owned = vfio_dev_in_groups(&cur_vma->vdev, groups);
2545 }
2546
2547 if (!owned) {
2548 ret = -EINVAL;
2549 goto err_undo;
2550 }
2551
2552 /*
2553 * Locking multiple devices is prone to deadlock, runaway and
2554 * unwind if we hit contention.
2555 */
2556 if (!vfio_pci_zap_and_vma_lock(cur_vma, true)) {
2557 ret = -EBUSY;
2558 goto err_undo;
2559 }
2560 }
2561 cur_vma = NULL;
2562
2563 list_for_each_entry(cur_mem, &dev_set->device_list, vdev.dev_set_list) {
2564 if (!down_write_trylock(&cur_mem->memory_lock)) {
2565 ret = -EBUSY;
2566 goto err_undo;
2567 }
2568 mutex_unlock(&cur_mem->vma_lock);
2569 }
2570 cur_mem = NULL;
2571
2572 /*
2573 * The pci_reset_bus() will reset all the devices in the bus.
2574 * The power state can be non-D0 for some of the devices in the bus.
2575 * For these devices, the pci_reset_bus() will internally set
2576 * the power state to D0 without vfio driver involvement.
2577 * For the devices which have NoSoftRst-, the reset function can
2578 * cause the PCI config space reset without restoring the original
2579 * state (saved locally in 'vdev->pm_save').
2580 */
2581 list_for_each_entry(cur, &dev_set->device_list, vdev.dev_set_list)
2582 vfio_pci_set_power_state(cur, PCI_D0);
2583
2584 ret = pci_reset_bus(pdev);
2585
2586 err_undo:
2587 list_for_each_entry(cur, &dev_set->device_list, vdev.dev_set_list) {
2588 if (cur == cur_mem)
2589 is_mem = false;
2590 if (cur == cur_vma)
2591 break;
2592 if (is_mem)
2593 up_write(&cur->memory_lock);
2594 else
2595 mutex_unlock(&cur->vma_lock);
2596 }
2597
2598 list_for_each_entry(cur, &dev_set->device_list, vdev.dev_set_list)
2599 pm_runtime_put(&cur->pdev->dev);
2600 err_unlock:
2601 mutex_unlock(&dev_set->lock);
2602 return ret;
2603 }
2604
vfio_pci_dev_set_needs_reset(struct vfio_device_set * dev_set)2605 static bool vfio_pci_dev_set_needs_reset(struct vfio_device_set *dev_set)
2606 {
2607 struct vfio_pci_core_device *cur;
2608 bool needs_reset = false;
2609
2610 /* No other VFIO device in the set can be open. */
2611 if (vfio_device_set_open_count(dev_set) > 1)
2612 return false;
2613
2614 list_for_each_entry(cur, &dev_set->device_list, vdev.dev_set_list)
2615 needs_reset |= cur->needs_reset;
2616 return needs_reset;
2617 }
2618
2619 /*
2620 * If a bus or slot reset is available for the provided dev_set and:
2621 * - All of the devices affected by that bus or slot reset are unused
2622 * - At least one of the affected devices is marked dirty via
2623 * needs_reset (such as by lack of FLR support)
2624 * Then attempt to perform that bus or slot reset.
2625 */
vfio_pci_dev_set_try_reset(struct vfio_device_set * dev_set)2626 static void vfio_pci_dev_set_try_reset(struct vfio_device_set *dev_set)
2627 {
2628 struct vfio_pci_core_device *cur;
2629 struct pci_dev *pdev;
2630 bool reset_done = false;
2631
2632 if (!vfio_pci_dev_set_needs_reset(dev_set))
2633 return;
2634
2635 pdev = vfio_pci_dev_set_resettable(dev_set);
2636 if (!pdev)
2637 return;
2638
2639 /*
2640 * Some of the devices in the bus can be in the runtime suspended
2641 * state. Increment the usage count for all the devices in the dev_set
2642 * before reset and decrement the same after reset.
2643 */
2644 if (!disable_idle_d3 && vfio_pci_dev_set_pm_runtime_get(dev_set))
2645 return;
2646
2647 if (!pci_reset_bus(pdev))
2648 reset_done = true;
2649
2650 list_for_each_entry(cur, &dev_set->device_list, vdev.dev_set_list) {
2651 if (reset_done)
2652 cur->needs_reset = false;
2653
2654 if (!disable_idle_d3)
2655 pm_runtime_put(&cur->pdev->dev);
2656 }
2657 }
2658
vfio_pci_core_set_params(bool is_nointxmask,bool is_disable_vga,bool is_disable_idle_d3)2659 void vfio_pci_core_set_params(bool is_nointxmask, bool is_disable_vga,
2660 bool is_disable_idle_d3)
2661 {
2662 nointxmask = is_nointxmask;
2663 disable_vga = is_disable_vga;
2664 disable_idle_d3 = is_disable_idle_d3;
2665 }
2666 EXPORT_SYMBOL_GPL(vfio_pci_core_set_params);
2667
vfio_pci_core_cleanup(void)2668 static void vfio_pci_core_cleanup(void)
2669 {
2670 vfio_pci_uninit_perm_bits();
2671 }
2672
vfio_pci_core_init(void)2673 static int __init vfio_pci_core_init(void)
2674 {
2675 /* Allocate shared config space permission data used by all devices */
2676 return vfio_pci_init_perm_bits();
2677 }
2678
2679 module_init(vfio_pci_core_init);
2680 module_exit(vfio_pci_core_cleanup);
2681
2682 MODULE_LICENSE("GPL v2");
2683 MODULE_AUTHOR(DRIVER_AUTHOR);
2684 MODULE_DESCRIPTION(DRIVER_DESC);
2685