1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Volume Management Device driver
4 * Copyright (c) 2015, Intel Corporation.
5 */
6
7 #include <linux/device.h>
8 #include <linux/interrupt.h>
9 #include <linux/irq.h>
10 #include <linux/kernel.h>
11 #include <linux/module.h>
12 #include <linux/msi.h>
13 #include <linux/pci.h>
14 #include <linux/srcu.h>
15 #include <linux/rculist.h>
16 #include <linux/rcupdate.h>
17
18 #include <asm/irqdomain.h>
19 #include <asm/device.h>
20 #include <asm/msi.h>
21 #include <asm/msidef.h>
22
23 #define VMD_CFGBAR 0
24 #define VMD_MEMBAR1 2
25 #define VMD_MEMBAR2 4
26
27 #define PCI_REG_VMCAP 0x40
28 #define BUS_RESTRICT_CAP(vmcap) (vmcap & 0x1)
29 #define PCI_REG_VMCONFIG 0x44
30 #define BUS_RESTRICT_CFG(vmcfg) ((vmcfg >> 8) & 0x3)
31 #define PCI_REG_VMLOCK 0x70
32 #define MB2_SHADOW_EN(vmlock) (vmlock & 0x2)
33
34 #define MB2_SHADOW_OFFSET 0x2000
35 #define MB2_SHADOW_SIZE 16
36
37 enum vmd_features {
38 /*
39 * Device may contain registers which hint the physical location of the
40 * membars, in order to allow proper address translation during
41 * resource assignment to enable guest virtualization
42 */
43 VMD_FEAT_HAS_MEMBAR_SHADOW = (1 << 0),
44
45 /*
46 * Device may provide root port configuration information which limits
47 * bus numbering
48 */
49 VMD_FEAT_HAS_BUS_RESTRICTIONS = (1 << 1),
50
51 /*
52 * Device contains physical location shadow registers in
53 * vendor-specific capability space
54 */
55 VMD_FEAT_HAS_MEMBAR_SHADOW_VSCAP = (1 << 2),
56 };
57
58 /*
59 * Lock for manipulating VMD IRQ lists.
60 */
61 static DEFINE_RAW_SPINLOCK(list_lock);
62
63 /**
64 * struct vmd_irq - private data to map driver IRQ to the VMD shared vector
65 * @node: list item for parent traversal.
66 * @irq: back pointer to parent.
67 * @enabled: true if driver enabled IRQ
68 * @virq: the virtual IRQ value provided to the requesting driver.
69 *
70 * Every MSI/MSI-X IRQ requested for a device in a VMD domain will be mapped to
71 * a VMD IRQ using this structure.
72 */
73 struct vmd_irq {
74 struct list_head node;
75 struct vmd_irq_list *irq;
76 bool enabled;
77 unsigned int virq;
78 };
79
80 /**
81 * struct vmd_irq_list - list of driver requested IRQs mapping to a VMD vector
82 * @irq_list: the list of irq's the VMD one demuxes to.
83 * @srcu: SRCU struct for local synchronization.
84 * @count: number of child IRQs assigned to this vector; used to track
85 * sharing.
86 */
87 struct vmd_irq_list {
88 struct list_head irq_list;
89 struct srcu_struct srcu;
90 unsigned int count;
91 };
92
93 struct vmd_dev {
94 struct pci_dev *dev;
95
96 spinlock_t cfg_lock;
97 char __iomem *cfgbar;
98
99 int msix_count;
100 struct vmd_irq_list *irqs;
101
102 struct pci_sysdata sysdata;
103 struct resource resources[3];
104 struct irq_domain *irq_domain;
105 struct pci_bus *bus;
106 u8 busn_start;
107 };
108
vmd_from_bus(struct pci_bus * bus)109 static inline struct vmd_dev *vmd_from_bus(struct pci_bus *bus)
110 {
111 return container_of(bus->sysdata, struct vmd_dev, sysdata);
112 }
113
index_from_irqs(struct vmd_dev * vmd,struct vmd_irq_list * irqs)114 static inline unsigned int index_from_irqs(struct vmd_dev *vmd,
115 struct vmd_irq_list *irqs)
116 {
117 return irqs - vmd->irqs;
118 }
119
120 /*
121 * Drivers managing a device in a VMD domain allocate their own IRQs as before,
122 * but the MSI entry for the hardware it's driving will be programmed with a
123 * destination ID for the VMD MSI-X table. The VMD muxes interrupts in its
124 * domain into one of its own, and the VMD driver de-muxes these for the
125 * handlers sharing that VMD IRQ. The vmd irq_domain provides the operations
126 * and irq_chip to set this up.
127 */
vmd_compose_msi_msg(struct irq_data * data,struct msi_msg * msg)128 static void vmd_compose_msi_msg(struct irq_data *data, struct msi_msg *msg)
129 {
130 struct vmd_irq *vmdirq = data->chip_data;
131 struct vmd_irq_list *irq = vmdirq->irq;
132 struct vmd_dev *vmd = irq_data_get_irq_handler_data(data);
133
134 msg->address_hi = MSI_ADDR_BASE_HI;
135 msg->address_lo = MSI_ADDR_BASE_LO |
136 MSI_ADDR_DEST_ID(index_from_irqs(vmd, irq));
137 msg->data = 0;
138 }
139
140 /*
141 * We rely on MSI_FLAG_USE_DEF_CHIP_OPS to set the IRQ mask/unmask ops.
142 */
vmd_irq_enable(struct irq_data * data)143 static void vmd_irq_enable(struct irq_data *data)
144 {
145 struct vmd_irq *vmdirq = data->chip_data;
146 unsigned long flags;
147
148 raw_spin_lock_irqsave(&list_lock, flags);
149 WARN_ON(vmdirq->enabled);
150 list_add_tail_rcu(&vmdirq->node, &vmdirq->irq->irq_list);
151 vmdirq->enabled = true;
152 raw_spin_unlock_irqrestore(&list_lock, flags);
153
154 data->chip->irq_unmask(data);
155 }
156
vmd_irq_disable(struct irq_data * data)157 static void vmd_irq_disable(struct irq_data *data)
158 {
159 struct vmd_irq *vmdirq = data->chip_data;
160 unsigned long flags;
161
162 data->chip->irq_mask(data);
163
164 raw_spin_lock_irqsave(&list_lock, flags);
165 if (vmdirq->enabled) {
166 list_del_rcu(&vmdirq->node);
167 vmdirq->enabled = false;
168 }
169 raw_spin_unlock_irqrestore(&list_lock, flags);
170 }
171
172 /*
173 * XXX: Stubbed until we develop acceptable way to not create conflicts with
174 * other devices sharing the same vector.
175 */
vmd_irq_set_affinity(struct irq_data * data,const struct cpumask * dest,bool force)176 static int vmd_irq_set_affinity(struct irq_data *data,
177 const struct cpumask *dest, bool force)
178 {
179 return -EINVAL;
180 }
181
182 static struct irq_chip vmd_msi_controller = {
183 .name = "VMD-MSI",
184 .irq_enable = vmd_irq_enable,
185 .irq_disable = vmd_irq_disable,
186 .irq_compose_msi_msg = vmd_compose_msi_msg,
187 .irq_set_affinity = vmd_irq_set_affinity,
188 };
189
vmd_get_hwirq(struct msi_domain_info * info,msi_alloc_info_t * arg)190 static irq_hw_number_t vmd_get_hwirq(struct msi_domain_info *info,
191 msi_alloc_info_t *arg)
192 {
193 return 0;
194 }
195
196 /*
197 * XXX: We can be even smarter selecting the best IRQ once we solve the
198 * affinity problem.
199 */
vmd_next_irq(struct vmd_dev * vmd,struct msi_desc * desc)200 static struct vmd_irq_list *vmd_next_irq(struct vmd_dev *vmd, struct msi_desc *desc)
201 {
202 int i, best = 1;
203 unsigned long flags;
204
205 if (vmd->msix_count == 1)
206 return &vmd->irqs[0];
207
208 /*
209 * White list for fast-interrupt handlers. All others will share the
210 * "slow" interrupt vector.
211 */
212 switch (msi_desc_to_pci_dev(desc)->class) {
213 case PCI_CLASS_STORAGE_EXPRESS:
214 break;
215 default:
216 return &vmd->irqs[0];
217 }
218
219 raw_spin_lock_irqsave(&list_lock, flags);
220 for (i = 1; i < vmd->msix_count; i++)
221 if (vmd->irqs[i].count < vmd->irqs[best].count)
222 best = i;
223 vmd->irqs[best].count++;
224 raw_spin_unlock_irqrestore(&list_lock, flags);
225
226 return &vmd->irqs[best];
227 }
228
vmd_msi_init(struct irq_domain * domain,struct msi_domain_info * info,unsigned int virq,irq_hw_number_t hwirq,msi_alloc_info_t * arg)229 static int vmd_msi_init(struct irq_domain *domain, struct msi_domain_info *info,
230 unsigned int virq, irq_hw_number_t hwirq,
231 msi_alloc_info_t *arg)
232 {
233 struct msi_desc *desc = arg->desc;
234 struct vmd_dev *vmd = vmd_from_bus(msi_desc_to_pci_dev(desc)->bus);
235 struct vmd_irq *vmdirq = kzalloc(sizeof(*vmdirq), GFP_KERNEL);
236 unsigned int index, vector;
237
238 if (!vmdirq)
239 return -ENOMEM;
240
241 INIT_LIST_HEAD(&vmdirq->node);
242 vmdirq->irq = vmd_next_irq(vmd, desc);
243 vmdirq->virq = virq;
244 index = index_from_irqs(vmd, vmdirq->irq);
245 vector = pci_irq_vector(vmd->dev, index);
246
247 irq_domain_set_info(domain, virq, vector, info->chip, vmdirq,
248 handle_untracked_irq, vmd, NULL);
249 return 0;
250 }
251
vmd_msi_free(struct irq_domain * domain,struct msi_domain_info * info,unsigned int virq)252 static void vmd_msi_free(struct irq_domain *domain,
253 struct msi_domain_info *info, unsigned int virq)
254 {
255 struct vmd_irq *vmdirq = irq_get_chip_data(virq);
256 unsigned long flags;
257
258 synchronize_srcu(&vmdirq->irq->srcu);
259
260 /* XXX: Potential optimization to rebalance */
261 raw_spin_lock_irqsave(&list_lock, flags);
262 vmdirq->irq->count--;
263 raw_spin_unlock_irqrestore(&list_lock, flags);
264
265 kfree(vmdirq);
266 }
267
vmd_msi_prepare(struct irq_domain * domain,struct device * dev,int nvec,msi_alloc_info_t * arg)268 static int vmd_msi_prepare(struct irq_domain *domain, struct device *dev,
269 int nvec, msi_alloc_info_t *arg)
270 {
271 struct pci_dev *pdev = to_pci_dev(dev);
272 struct vmd_dev *vmd = vmd_from_bus(pdev->bus);
273
274 if (nvec > vmd->msix_count)
275 return vmd->msix_count;
276
277 memset(arg, 0, sizeof(*arg));
278 return 0;
279 }
280
vmd_set_desc(msi_alloc_info_t * arg,struct msi_desc * desc)281 static void vmd_set_desc(msi_alloc_info_t *arg, struct msi_desc *desc)
282 {
283 arg->desc = desc;
284 }
285
286 static struct msi_domain_ops vmd_msi_domain_ops = {
287 .get_hwirq = vmd_get_hwirq,
288 .msi_init = vmd_msi_init,
289 .msi_free = vmd_msi_free,
290 .msi_prepare = vmd_msi_prepare,
291 .set_desc = vmd_set_desc,
292 };
293
294 static struct msi_domain_info vmd_msi_domain_info = {
295 .flags = MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS |
296 MSI_FLAG_PCI_MSIX,
297 .ops = &vmd_msi_domain_ops,
298 .chip = &vmd_msi_controller,
299 };
300
vmd_create_irq_domain(struct vmd_dev * vmd)301 static int vmd_create_irq_domain(struct vmd_dev *vmd)
302 {
303 struct fwnode_handle *fn;
304
305 fn = irq_domain_alloc_named_id_fwnode("VMD-MSI", vmd->sysdata.domain);
306 if (!fn)
307 return -ENODEV;
308
309 vmd->irq_domain = pci_msi_create_irq_domain(fn, &vmd_msi_domain_info, NULL);
310 if (!vmd->irq_domain) {
311 irq_domain_free_fwnode(fn);
312 return -ENODEV;
313 }
314
315 return 0;
316 }
317
vmd_remove_irq_domain(struct vmd_dev * vmd)318 static void vmd_remove_irq_domain(struct vmd_dev *vmd)
319 {
320 if (vmd->irq_domain) {
321 struct fwnode_handle *fn = vmd->irq_domain->fwnode;
322
323 irq_domain_remove(vmd->irq_domain);
324 irq_domain_free_fwnode(fn);
325 }
326 }
327
vmd_cfg_addr(struct vmd_dev * vmd,struct pci_bus * bus,unsigned int devfn,int reg,int len)328 static char __iomem *vmd_cfg_addr(struct vmd_dev *vmd, struct pci_bus *bus,
329 unsigned int devfn, int reg, int len)
330 {
331 char __iomem *addr = vmd->cfgbar +
332 ((bus->number - vmd->busn_start) << 20) +
333 (devfn << 12) + reg;
334
335 if ((addr - vmd->cfgbar) + len >=
336 resource_size(&vmd->dev->resource[VMD_CFGBAR]))
337 return NULL;
338
339 return addr;
340 }
341
342 /*
343 * CPU may deadlock if config space is not serialized on some versions of this
344 * hardware, so all config space access is done under a spinlock.
345 */
vmd_pci_read(struct pci_bus * bus,unsigned int devfn,int reg,int len,u32 * value)346 static int vmd_pci_read(struct pci_bus *bus, unsigned int devfn, int reg,
347 int len, u32 *value)
348 {
349 struct vmd_dev *vmd = vmd_from_bus(bus);
350 char __iomem *addr = vmd_cfg_addr(vmd, bus, devfn, reg, len);
351 unsigned long flags;
352 int ret = 0;
353
354 if (!addr)
355 return -EFAULT;
356
357 spin_lock_irqsave(&vmd->cfg_lock, flags);
358 switch (len) {
359 case 1:
360 *value = readb(addr);
361 break;
362 case 2:
363 *value = readw(addr);
364 break;
365 case 4:
366 *value = readl(addr);
367 break;
368 default:
369 ret = -EINVAL;
370 break;
371 }
372 spin_unlock_irqrestore(&vmd->cfg_lock, flags);
373 return ret;
374 }
375
376 /*
377 * VMD h/w converts non-posted config writes to posted memory writes. The
378 * read-back in this function forces the completion so it returns only after
379 * the config space was written, as expected.
380 */
vmd_pci_write(struct pci_bus * bus,unsigned int devfn,int reg,int len,u32 value)381 static int vmd_pci_write(struct pci_bus *bus, unsigned int devfn, int reg,
382 int len, u32 value)
383 {
384 struct vmd_dev *vmd = vmd_from_bus(bus);
385 char __iomem *addr = vmd_cfg_addr(vmd, bus, devfn, reg, len);
386 unsigned long flags;
387 int ret = 0;
388
389 if (!addr)
390 return -EFAULT;
391
392 spin_lock_irqsave(&vmd->cfg_lock, flags);
393 switch (len) {
394 case 1:
395 writeb(value, addr);
396 readb(addr);
397 break;
398 case 2:
399 writew(value, addr);
400 readw(addr);
401 break;
402 case 4:
403 writel(value, addr);
404 readl(addr);
405 break;
406 default:
407 ret = -EINVAL;
408 break;
409 }
410 spin_unlock_irqrestore(&vmd->cfg_lock, flags);
411 return ret;
412 }
413
414 static struct pci_ops vmd_ops = {
415 .read = vmd_pci_read,
416 .write = vmd_pci_write,
417 };
418
vmd_attach_resources(struct vmd_dev * vmd)419 static void vmd_attach_resources(struct vmd_dev *vmd)
420 {
421 vmd->dev->resource[VMD_MEMBAR1].child = &vmd->resources[1];
422 vmd->dev->resource[VMD_MEMBAR2].child = &vmd->resources[2];
423 }
424
vmd_detach_resources(struct vmd_dev * vmd)425 static void vmd_detach_resources(struct vmd_dev *vmd)
426 {
427 vmd->dev->resource[VMD_MEMBAR1].child = NULL;
428 vmd->dev->resource[VMD_MEMBAR2].child = NULL;
429 }
430
431 /*
432 * VMD domains start at 0x10000 to not clash with ACPI _SEG domains.
433 * Per ACPI r6.0, sec 6.5.6, _SEG returns an integer, of which the lower
434 * 16 bits are the PCI Segment Group (domain) number. Other bits are
435 * currently reserved.
436 */
vmd_find_free_domain(void)437 static int vmd_find_free_domain(void)
438 {
439 int domain = 0xffff;
440 struct pci_bus *bus = NULL;
441
442 while ((bus = pci_find_next_bus(bus)) != NULL)
443 domain = max_t(int, domain, pci_domain_nr(bus));
444 return domain + 1;
445 }
446
vmd_get_phys_offsets(struct vmd_dev * vmd,bool native_hint,resource_size_t * offset1,resource_size_t * offset2)447 static int vmd_get_phys_offsets(struct vmd_dev *vmd, bool native_hint,
448 resource_size_t *offset1,
449 resource_size_t *offset2)
450 {
451 struct pci_dev *dev = vmd->dev;
452 u64 phys1, phys2;
453
454 if (native_hint) {
455 u32 vmlock;
456 int ret;
457
458 ret = pci_read_config_dword(dev, PCI_REG_VMLOCK, &vmlock);
459 if (ret || vmlock == ~0)
460 return -ENODEV;
461
462 if (MB2_SHADOW_EN(vmlock)) {
463 void __iomem *membar2;
464
465 membar2 = pci_iomap(dev, VMD_MEMBAR2, 0);
466 if (!membar2)
467 return -ENOMEM;
468 phys1 = readq(membar2 + MB2_SHADOW_OFFSET);
469 phys2 = readq(membar2 + MB2_SHADOW_OFFSET + 8);
470 pci_iounmap(dev, membar2);
471 } else
472 return 0;
473 } else {
474 /* Hypervisor-Emulated Vendor-Specific Capability */
475 int pos = pci_find_capability(dev, PCI_CAP_ID_VNDR);
476 u32 reg, regu;
477
478 pci_read_config_dword(dev, pos + 4, ®);
479
480 /* "SHDW" */
481 if (pos && reg == 0x53484457) {
482 pci_read_config_dword(dev, pos + 8, ®);
483 pci_read_config_dword(dev, pos + 12, ®u);
484 phys1 = (u64) regu << 32 | reg;
485
486 pci_read_config_dword(dev, pos + 16, ®);
487 pci_read_config_dword(dev, pos + 20, ®u);
488 phys2 = (u64) regu << 32 | reg;
489 } else
490 return 0;
491 }
492
493 *offset1 = dev->resource[VMD_MEMBAR1].start -
494 (phys1 & PCI_BASE_ADDRESS_MEM_MASK);
495 *offset2 = dev->resource[VMD_MEMBAR2].start -
496 (phys2 & PCI_BASE_ADDRESS_MEM_MASK);
497
498 return 0;
499 }
500
vmd_get_bus_number_start(struct vmd_dev * vmd)501 static int vmd_get_bus_number_start(struct vmd_dev *vmd)
502 {
503 struct pci_dev *dev = vmd->dev;
504 u16 reg;
505
506 pci_read_config_word(dev, PCI_REG_VMCAP, ®);
507 if (BUS_RESTRICT_CAP(reg)) {
508 pci_read_config_word(dev, PCI_REG_VMCONFIG, ®);
509
510 switch (BUS_RESTRICT_CFG(reg)) {
511 case 0:
512 vmd->busn_start = 0;
513 break;
514 case 1:
515 vmd->busn_start = 128;
516 break;
517 case 2:
518 vmd->busn_start = 224;
519 break;
520 default:
521 pci_err(dev, "Unknown Bus Offset Setting (%d)\n",
522 BUS_RESTRICT_CFG(reg));
523 return -ENODEV;
524 }
525 }
526
527 return 0;
528 }
529
vmd_irq(int irq,void * data)530 static irqreturn_t vmd_irq(int irq, void *data)
531 {
532 struct vmd_irq_list *irqs = data;
533 struct vmd_irq *vmdirq;
534 int idx;
535
536 idx = srcu_read_lock(&irqs->srcu);
537 list_for_each_entry_rcu(vmdirq, &irqs->irq_list, node)
538 generic_handle_irq(vmdirq->virq);
539 srcu_read_unlock(&irqs->srcu, idx);
540
541 return IRQ_HANDLED;
542 }
543
vmd_alloc_irqs(struct vmd_dev * vmd)544 static int vmd_alloc_irqs(struct vmd_dev *vmd)
545 {
546 struct pci_dev *dev = vmd->dev;
547 int i, err;
548
549 vmd->msix_count = pci_msix_vec_count(dev);
550 if (vmd->msix_count < 0)
551 return -ENODEV;
552
553 vmd->msix_count = pci_alloc_irq_vectors(dev, 1, vmd->msix_count,
554 PCI_IRQ_MSIX);
555 if (vmd->msix_count < 0)
556 return vmd->msix_count;
557
558 vmd->irqs = devm_kcalloc(&dev->dev, vmd->msix_count, sizeof(*vmd->irqs),
559 GFP_KERNEL);
560 if (!vmd->irqs)
561 return -ENOMEM;
562
563 for (i = 0; i < vmd->msix_count; i++) {
564 err = init_srcu_struct(&vmd->irqs[i].srcu);
565 if (err)
566 return err;
567
568 INIT_LIST_HEAD(&vmd->irqs[i].irq_list);
569 err = devm_request_irq(&dev->dev, pci_irq_vector(dev, i),
570 vmd_irq, IRQF_NO_THREAD,
571 "vmd", &vmd->irqs[i]);
572 if (err)
573 return err;
574 }
575
576 return 0;
577 }
578
vmd_enable_domain(struct vmd_dev * vmd,unsigned long features)579 static int vmd_enable_domain(struct vmd_dev *vmd, unsigned long features)
580 {
581 struct pci_sysdata *sd = &vmd->sysdata;
582 struct resource *res;
583 u32 upper_bits;
584 unsigned long flags;
585 LIST_HEAD(resources);
586 resource_size_t offset[2] = {0};
587 resource_size_t membar2_offset = 0x2000;
588 struct pci_bus *child;
589 int ret;
590
591 /*
592 * Shadow registers may exist in certain VMD device ids which allow
593 * guests to correctly assign host physical addresses to the root ports
594 * and child devices. These registers will either return the host value
595 * or 0, depending on an enable bit in the VMD device.
596 */
597 if (features & VMD_FEAT_HAS_MEMBAR_SHADOW) {
598 membar2_offset = MB2_SHADOW_OFFSET + MB2_SHADOW_SIZE;
599 ret = vmd_get_phys_offsets(vmd, true, &offset[0], &offset[1]);
600 if (ret)
601 return ret;
602 } else if (features & VMD_FEAT_HAS_MEMBAR_SHADOW_VSCAP) {
603 ret = vmd_get_phys_offsets(vmd, false, &offset[0], &offset[1]);
604 if (ret)
605 return ret;
606 }
607
608 /*
609 * Certain VMD devices may have a root port configuration option which
610 * limits the bus range to between 0-127, 128-255, or 224-255
611 */
612 if (features & VMD_FEAT_HAS_BUS_RESTRICTIONS) {
613 ret = vmd_get_bus_number_start(vmd);
614 if (ret)
615 return ret;
616 }
617
618 res = &vmd->dev->resource[VMD_CFGBAR];
619 vmd->resources[0] = (struct resource) {
620 .name = "VMD CFGBAR",
621 .start = vmd->busn_start,
622 .end = vmd->busn_start + (resource_size(res) >> 20) - 1,
623 .flags = IORESOURCE_BUS | IORESOURCE_PCI_FIXED,
624 };
625
626 /*
627 * If the window is below 4GB, clear IORESOURCE_MEM_64 so we can
628 * put 32-bit resources in the window.
629 *
630 * There's no hardware reason why a 64-bit window *couldn't*
631 * contain a 32-bit resource, but pbus_size_mem() computes the
632 * bridge window size assuming a 64-bit window will contain no
633 * 32-bit resources. __pci_assign_resource() enforces that
634 * artificial restriction to make sure everything will fit.
635 *
636 * The only way we could use a 64-bit non-prefetchable MEMBAR is
637 * if its address is <4GB so that we can convert it to a 32-bit
638 * resource. To be visible to the host OS, all VMD endpoints must
639 * be initially configured by platform BIOS, which includes setting
640 * up these resources. We can assume the device is configured
641 * according to the platform needs.
642 */
643 res = &vmd->dev->resource[VMD_MEMBAR1];
644 upper_bits = upper_32_bits(res->end);
645 flags = res->flags & ~IORESOURCE_SIZEALIGN;
646 if (!upper_bits)
647 flags &= ~IORESOURCE_MEM_64;
648 vmd->resources[1] = (struct resource) {
649 .name = "VMD MEMBAR1",
650 .start = res->start,
651 .end = res->end,
652 .flags = flags,
653 .parent = res,
654 };
655
656 res = &vmd->dev->resource[VMD_MEMBAR2];
657 upper_bits = upper_32_bits(res->end);
658 flags = res->flags & ~IORESOURCE_SIZEALIGN;
659 if (!upper_bits)
660 flags &= ~IORESOURCE_MEM_64;
661 vmd->resources[2] = (struct resource) {
662 .name = "VMD MEMBAR2",
663 .start = res->start + membar2_offset,
664 .end = res->end,
665 .flags = flags,
666 .parent = res,
667 };
668
669 sd->vmd_dev = vmd->dev;
670 sd->domain = vmd_find_free_domain();
671 if (sd->domain < 0)
672 return sd->domain;
673
674 sd->node = pcibus_to_node(vmd->dev->bus);
675
676 ret = vmd_create_irq_domain(vmd);
677 if (ret)
678 return ret;
679
680 /*
681 * Override the irq domain bus token so the domain can be distinguished
682 * from a regular PCI/MSI domain.
683 */
684 irq_domain_update_bus_token(vmd->irq_domain, DOMAIN_BUS_VMD_MSI);
685
686 pci_add_resource(&resources, &vmd->resources[0]);
687 pci_add_resource_offset(&resources, &vmd->resources[1], offset[0]);
688 pci_add_resource_offset(&resources, &vmd->resources[2], offset[1]);
689
690 vmd->bus = pci_create_root_bus(&vmd->dev->dev, vmd->busn_start,
691 &vmd_ops, sd, &resources);
692 if (!vmd->bus) {
693 pci_free_resource_list(&resources);
694 vmd_remove_irq_domain(vmd);
695 return -ENODEV;
696 }
697
698 vmd_attach_resources(vmd);
699 if (vmd->irq_domain)
700 dev_set_msi_domain(&vmd->bus->dev, vmd->irq_domain);
701
702 pci_scan_child_bus(vmd->bus);
703 pci_assign_unassigned_bus_resources(vmd->bus);
704
705 /*
706 * VMD root buses are virtual and don't return true on pci_is_pcie()
707 * and will fail pcie_bus_configure_settings() early. It can instead be
708 * run on each of the real root ports.
709 */
710 list_for_each_entry(child, &vmd->bus->children, node)
711 pcie_bus_configure_settings(child);
712
713 pci_bus_add_devices(vmd->bus);
714
715 WARN(sysfs_create_link(&vmd->dev->dev.kobj, &vmd->bus->dev.kobj,
716 "domain"), "Can't create symlink to domain\n");
717 return 0;
718 }
719
vmd_probe(struct pci_dev * dev,const struct pci_device_id * id)720 static int vmd_probe(struct pci_dev *dev, const struct pci_device_id *id)
721 {
722 struct vmd_dev *vmd;
723 int err;
724
725 if (resource_size(&dev->resource[VMD_CFGBAR]) < (1 << 20))
726 return -ENOMEM;
727
728 vmd = devm_kzalloc(&dev->dev, sizeof(*vmd), GFP_KERNEL);
729 if (!vmd)
730 return -ENOMEM;
731
732 vmd->dev = dev;
733 err = pcim_enable_device(dev);
734 if (err < 0)
735 return err;
736
737 vmd->cfgbar = pcim_iomap(dev, VMD_CFGBAR, 0);
738 if (!vmd->cfgbar)
739 return -ENOMEM;
740
741 pci_set_master(dev);
742 if (dma_set_mask_and_coherent(&dev->dev, DMA_BIT_MASK(64)) &&
743 dma_set_mask_and_coherent(&dev->dev, DMA_BIT_MASK(32)))
744 return -ENODEV;
745
746 err = vmd_alloc_irqs(vmd);
747 if (err)
748 return err;
749
750 spin_lock_init(&vmd->cfg_lock);
751 pci_set_drvdata(dev, vmd);
752 err = vmd_enable_domain(vmd, (unsigned long) id->driver_data);
753 if (err)
754 return err;
755
756 dev_info(&vmd->dev->dev, "Bound to PCI domain %04x\n",
757 vmd->sysdata.domain);
758 return 0;
759 }
760
vmd_cleanup_srcu(struct vmd_dev * vmd)761 static void vmd_cleanup_srcu(struct vmd_dev *vmd)
762 {
763 int i;
764
765 for (i = 0; i < vmd->msix_count; i++)
766 cleanup_srcu_struct(&vmd->irqs[i].srcu);
767 }
768
vmd_remove(struct pci_dev * dev)769 static void vmd_remove(struct pci_dev *dev)
770 {
771 struct vmd_dev *vmd = pci_get_drvdata(dev);
772
773 sysfs_remove_link(&vmd->dev->dev.kobj, "domain");
774 pci_stop_root_bus(vmd->bus);
775 pci_remove_root_bus(vmd->bus);
776 vmd_cleanup_srcu(vmd);
777 vmd_detach_resources(vmd);
778 vmd_remove_irq_domain(vmd);
779 }
780
781 #ifdef CONFIG_PM_SLEEP
vmd_suspend(struct device * dev)782 static int vmd_suspend(struct device *dev)
783 {
784 struct pci_dev *pdev = to_pci_dev(dev);
785 struct vmd_dev *vmd = pci_get_drvdata(pdev);
786 int i;
787
788 for (i = 0; i < vmd->msix_count; i++)
789 devm_free_irq(dev, pci_irq_vector(pdev, i), &vmd->irqs[i]);
790
791 return 0;
792 }
793
vmd_resume(struct device * dev)794 static int vmd_resume(struct device *dev)
795 {
796 struct pci_dev *pdev = to_pci_dev(dev);
797 struct vmd_dev *vmd = pci_get_drvdata(pdev);
798 int err, i;
799
800 for (i = 0; i < vmd->msix_count; i++) {
801 err = devm_request_irq(dev, pci_irq_vector(pdev, i),
802 vmd_irq, IRQF_NO_THREAD,
803 "vmd", &vmd->irqs[i]);
804 if (err)
805 return err;
806 }
807
808 return 0;
809 }
810 #endif
811 static SIMPLE_DEV_PM_OPS(vmd_dev_pm_ops, vmd_suspend, vmd_resume);
812
813 static const struct pci_device_id vmd_ids[] = {
814 {PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_VMD_201D),
815 .driver_data = VMD_FEAT_HAS_MEMBAR_SHADOW_VSCAP,},
816 {PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_VMD_28C0),
817 .driver_data = VMD_FEAT_HAS_MEMBAR_SHADOW |
818 VMD_FEAT_HAS_BUS_RESTRICTIONS,},
819 {PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x467f),
820 .driver_data = VMD_FEAT_HAS_MEMBAR_SHADOW_VSCAP |
821 VMD_FEAT_HAS_BUS_RESTRICTIONS,},
822 {PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x4c3d),
823 .driver_data = VMD_FEAT_HAS_MEMBAR_SHADOW_VSCAP |
824 VMD_FEAT_HAS_BUS_RESTRICTIONS,},
825 {PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_VMD_9A0B),
826 .driver_data = VMD_FEAT_HAS_MEMBAR_SHADOW_VSCAP |
827 VMD_FEAT_HAS_BUS_RESTRICTIONS,},
828 {0,}
829 };
830 MODULE_DEVICE_TABLE(pci, vmd_ids);
831
832 static struct pci_driver vmd_drv = {
833 .name = "vmd",
834 .id_table = vmd_ids,
835 .probe = vmd_probe,
836 .remove = vmd_remove,
837 .driver = {
838 .pm = &vmd_dev_pm_ops,
839 },
840 };
841 module_pci_driver(vmd_drv);
842
843 MODULE_AUTHOR("Intel Corporation");
844 MODULE_LICENSE("GPL v2");
845 MODULE_VERSION("0.6");
846