Lines Matching full:iommu
13 #include <linux/intel-iommu.h>
33 struct intel_iommu *iommu; member
40 struct intel_iommu *iommu; member
47 struct intel_iommu *iommu; member
74 * ->iommu->register_lock
83 static void iommu_disable_irq_remapping(struct intel_iommu *iommu);
86 static bool ir_pre_enabled(struct intel_iommu *iommu) in ir_pre_enabled() argument
88 return (iommu->flags & VTD_FLAG_IRQ_REMAP_PRE_ENABLED); in ir_pre_enabled()
91 static void clear_ir_pre_enabled(struct intel_iommu *iommu) in clear_ir_pre_enabled() argument
93 iommu->flags &= ~VTD_FLAG_IRQ_REMAP_PRE_ENABLED; in clear_ir_pre_enabled()
96 static void init_ir_status(struct intel_iommu *iommu) in init_ir_status() argument
100 gsts = readl(iommu->reg + DMAR_GSTS_REG); in init_ir_status()
102 iommu->flags |= VTD_FLAG_IRQ_REMAP_PRE_ENABLED; in init_ir_status()
105 static int alloc_irte(struct intel_iommu *iommu, in alloc_irte() argument
108 struct ir_table *table = iommu->ir_table; in alloc_irte()
121 if (mask > ecap_max_handle_mask(iommu->ecap)) { in alloc_irte()
124 ecap_max_handle_mask(iommu->ecap)); in alloc_irte()
132 pr_warn("IR%d: can't allocate an IRTE\n", iommu->seq_id); in alloc_irte()
134 irq_iommu->iommu = iommu; in alloc_irte()
145 static int qi_flush_iec(struct intel_iommu *iommu, int index, int mask) in qi_flush_iec() argument
155 return qi_submit_sync(iommu, &desc, 1, 0); in qi_flush_iec()
161 struct intel_iommu *iommu; in modify_irte() local
171 iommu = irq_iommu->iommu; in modify_irte()
174 irte = &iommu->ir_table->base[index]; in modify_irte()
196 __iommu_flush_cache(iommu, irte, sizeof(*irte)); in modify_irte()
198 rc = qi_flush_iec(iommu, index, 0); in modify_irte()
200 /* Update iommu mode according to the IRTE mode */ in modify_irte()
212 if (ir_hpet[i].id == hpet_id && ir_hpet[i].iommu) in map_hpet_to_ir()
213 return ir_hpet[i].iommu->ir_domain; in map_hpet_to_ir()
223 if (ir_ioapic[i].id == apic && ir_ioapic[i].iommu) in map_ioapic_to_iommu()
224 return ir_ioapic[i].iommu; in map_ioapic_to_iommu()
231 struct intel_iommu *iommu = map_ioapic_to_iommu(apic); in map_ioapic_to_ir() local
233 return iommu ? iommu->ir_domain : NULL; in map_ioapic_to_ir()
240 return drhd ? drhd->iommu->ir_msi_domain : NULL; in map_dev_to_ir()
246 struct intel_iommu *iommu; in clear_entries() local
252 iommu = irq_iommu->iommu; in clear_entries()
255 start = iommu->ir_table->base + index; in clear_entries()
262 bitmap_release_region(iommu->ir_table->bitmap, index, in clear_entries()
265 return qi_flush_iec(iommu, index, irq_iommu->irte_mask); in clear_entries()
325 if (ir_ioapic[i].iommu && ir_ioapic[i].id == apic) { in set_ioapic_sid()
352 if (ir_hpet[i].iommu && ir_hpet[i].id == id) { in set_hpet_sid()
438 static int iommu_load_old_irte(struct intel_iommu *iommu) in iommu_load_old_irte() argument
447 irta = dmar_readq(iommu->reg + DMAR_IRTA_REG); in iommu_load_old_irte()
461 memcpy(iommu->ir_table->base, old_ir_table, size); in iommu_load_old_irte()
463 __iommu_flush_cache(iommu, iommu->ir_table->base, size); in iommu_load_old_irte()
470 if (iommu->ir_table->base[i].present) in iommu_load_old_irte()
471 bitmap_set(iommu->ir_table->bitmap, i, 1); in iommu_load_old_irte()
480 static void iommu_set_irq_remapping(struct intel_iommu *iommu, int mode) in iommu_set_irq_remapping() argument
486 addr = virt_to_phys((void *)iommu->ir_table->base); in iommu_set_irq_remapping()
488 raw_spin_lock_irqsave(&iommu->register_lock, flags); in iommu_set_irq_remapping()
490 dmar_writeq(iommu->reg + DMAR_IRTA_REG, in iommu_set_irq_remapping()
494 writel(iommu->gcmd | DMA_GCMD_SIRTP, iommu->reg + DMAR_GCMD_REG); in iommu_set_irq_remapping()
496 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, in iommu_set_irq_remapping()
498 raw_spin_unlock_irqrestore(&iommu->register_lock, flags); in iommu_set_irq_remapping()
504 qi_global_iec(iommu); in iommu_set_irq_remapping()
507 static void iommu_enable_irq_remapping(struct intel_iommu *iommu) in iommu_enable_irq_remapping() argument
512 raw_spin_lock_irqsave(&iommu->register_lock, flags); in iommu_enable_irq_remapping()
515 iommu->gcmd |= DMA_GCMD_IRE; in iommu_enable_irq_remapping()
516 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG); in iommu_enable_irq_remapping()
517 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, in iommu_enable_irq_remapping()
522 iommu->gcmd &= ~DMA_GCMD_CFI; in iommu_enable_irq_remapping()
523 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG); in iommu_enable_irq_remapping()
524 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, in iommu_enable_irq_remapping()
538 raw_spin_unlock_irqrestore(&iommu->register_lock, flags); in iommu_enable_irq_remapping()
541 static int intel_setup_irq_remapping(struct intel_iommu *iommu) in intel_setup_irq_remapping() argument
548 if (iommu->ir_table) in intel_setup_irq_remapping()
555 pages = alloc_pages_node(iommu->node, GFP_KERNEL | __GFP_ZERO, in intel_setup_irq_remapping()
559 iommu->seq_id, INTR_REMAP_PAGE_ORDER); in intel_setup_irq_remapping()
565 pr_err("IR%d: failed to allocate bitmap\n", iommu->seq_id); in intel_setup_irq_remapping()
569 fn = irq_domain_alloc_named_id_fwnode("INTEL-IR", iommu->seq_id); in intel_setup_irq_remapping()
573 iommu->ir_domain = in intel_setup_irq_remapping()
577 iommu); in intel_setup_irq_remapping()
578 if (!iommu->ir_domain) { in intel_setup_irq_remapping()
579 pr_err("IR%d: failed to allocate irqdomain\n", iommu->seq_id); in intel_setup_irq_remapping()
582 iommu->ir_msi_domain = in intel_setup_irq_remapping()
583 arch_create_remap_msi_irq_domain(iommu->ir_domain, in intel_setup_irq_remapping()
585 iommu->seq_id); in intel_setup_irq_remapping()
589 iommu->ir_table = ir_table; in intel_setup_irq_remapping()
595 if (!iommu->qi) { in intel_setup_irq_remapping()
599 dmar_fault(-1, iommu); in intel_setup_irq_remapping()
600 dmar_disable_qi(iommu); in intel_setup_irq_remapping()
602 if (dmar_enable_qi(iommu)) { in intel_setup_irq_remapping()
608 init_ir_status(iommu); in intel_setup_irq_remapping()
610 if (ir_pre_enabled(iommu)) { in intel_setup_irq_remapping()
613 iommu->name); in intel_setup_irq_remapping()
614 clear_ir_pre_enabled(iommu); in intel_setup_irq_remapping()
615 iommu_disable_irq_remapping(iommu); in intel_setup_irq_remapping()
616 } else if (iommu_load_old_irte(iommu)) in intel_setup_irq_remapping()
618 iommu->name); in intel_setup_irq_remapping()
621 iommu->name); in intel_setup_irq_remapping()
624 iommu_set_irq_remapping(iommu, eim_mode); in intel_setup_irq_remapping()
629 if (iommu->ir_msi_domain) in intel_setup_irq_remapping()
630 irq_domain_remove(iommu->ir_msi_domain); in intel_setup_irq_remapping()
631 iommu->ir_msi_domain = NULL; in intel_setup_irq_remapping()
632 irq_domain_remove(iommu->ir_domain); in intel_setup_irq_remapping()
633 iommu->ir_domain = NULL; in intel_setup_irq_remapping()
643 iommu->ir_table = NULL; in intel_setup_irq_remapping()
648 static void intel_teardown_irq_remapping(struct intel_iommu *iommu) in intel_teardown_irq_remapping() argument
652 if (iommu && iommu->ir_table) { in intel_teardown_irq_remapping()
653 if (iommu->ir_msi_domain) { in intel_teardown_irq_remapping()
654 fn = iommu->ir_msi_domain->fwnode; in intel_teardown_irq_remapping()
656 irq_domain_remove(iommu->ir_msi_domain); in intel_teardown_irq_remapping()
658 iommu->ir_msi_domain = NULL; in intel_teardown_irq_remapping()
660 if (iommu->ir_domain) { in intel_teardown_irq_remapping()
661 fn = iommu->ir_domain->fwnode; in intel_teardown_irq_remapping()
663 irq_domain_remove(iommu->ir_domain); in intel_teardown_irq_remapping()
665 iommu->ir_domain = NULL; in intel_teardown_irq_remapping()
667 free_pages((unsigned long)iommu->ir_table->base, in intel_teardown_irq_remapping()
669 bitmap_free(iommu->ir_table->bitmap); in intel_teardown_irq_remapping()
670 kfree(iommu->ir_table); in intel_teardown_irq_remapping()
671 iommu->ir_table = NULL; in intel_teardown_irq_remapping()
678 static void iommu_disable_irq_remapping(struct intel_iommu *iommu) in iommu_disable_irq_remapping() argument
683 if (!ecap_ir_support(iommu->ecap)) in iommu_disable_irq_remapping()
690 qi_global_iec(iommu); in iommu_disable_irq_remapping()
692 raw_spin_lock_irqsave(&iommu->register_lock, flags); in iommu_disable_irq_remapping()
694 sts = readl(iommu->reg + DMAR_GSTS_REG); in iommu_disable_irq_remapping()
698 iommu->gcmd &= ~DMA_GCMD_IRE; in iommu_disable_irq_remapping()
699 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG); in iommu_disable_irq_remapping()
701 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, in iommu_disable_irq_remapping()
705 raw_spin_unlock_irqrestore(&iommu->register_lock, flags); in iommu_disable_irq_remapping()
720 struct intel_iommu *iommu; in intel_cleanup_irq_remapping() local
722 for_each_iommu(iommu, drhd) { in intel_cleanup_irq_remapping()
723 if (ecap_ir_support(iommu->ecap)) { in intel_cleanup_irq_remapping()
724 iommu_disable_irq_remapping(iommu); in intel_cleanup_irq_remapping()
725 intel_teardown_irq_remapping(iommu); in intel_cleanup_irq_remapping()
736 struct intel_iommu *iommu; in intel_prepare_irq_remapping() local
761 for_each_iommu(iommu, drhd) in intel_prepare_irq_remapping()
762 if (!ecap_ir_support(iommu->ecap)) in intel_prepare_irq_remapping()
774 for_each_iommu(iommu, drhd) { in intel_prepare_irq_remapping()
775 if (eim && !ecap_eim_support(iommu->ecap)) { in intel_prepare_irq_remapping()
776 pr_info("%s does not support EIM\n", iommu->name); in intel_prepare_irq_remapping()
786 for_each_iommu(iommu, drhd) { in intel_prepare_irq_remapping()
787 if (intel_setup_irq_remapping(iommu)) { in intel_prepare_irq_remapping()
789 iommu->name); in intel_prepare_irq_remapping()
807 struct intel_iommu *iommu; in set_irq_posting_cap() local
821 for_each_iommu(iommu, drhd) in set_irq_posting_cap()
822 if (!cap_pi_support(iommu->cap)) { in set_irq_posting_cap()
833 struct intel_iommu *iommu; in intel_enable_irq_remapping() local
839 for_each_iommu(iommu, drhd) { in intel_enable_irq_remapping()
840 if (!ir_pre_enabled(iommu)) in intel_enable_irq_remapping()
841 iommu_enable_irq_remapping(iommu); in intel_enable_irq_remapping()
862 struct intel_iommu *iommu, in ir_parse_one_hpet_scope() argument
885 if (ir_hpet[count].iommu == iommu && in ir_parse_one_hpet_scope()
888 else if (ir_hpet[count].iommu == NULL && free == -1) in ir_parse_one_hpet_scope()
896 ir_hpet[free].iommu = iommu; in ir_parse_one_hpet_scope()
907 struct intel_iommu *iommu, in ir_parse_one_ioapic_scope() argument
930 if (ir_ioapic[count].iommu == iommu && in ir_parse_one_ioapic_scope()
933 else if (ir_ioapic[count].iommu == NULL && free == -1) in ir_parse_one_ioapic_scope()
943 ir_ioapic[free].iommu = iommu; in ir_parse_one_ioapic_scope()
945 pr_info("IOAPIC id %d under DRHD base 0x%Lx IOMMU %d\n", in ir_parse_one_ioapic_scope()
946 scope->enumeration_id, drhd->address, iommu->seq_id); in ir_parse_one_ioapic_scope()
952 struct intel_iommu *iommu) in ir_parse_ioapic_hpet_scope() argument
966 ret = ir_parse_one_ioapic_scope(scope, iommu, drhd); in ir_parse_ioapic_hpet_scope()
968 ret = ir_parse_one_hpet_scope(scope, iommu, drhd); in ir_parse_ioapic_hpet_scope()
975 static void ir_remove_ioapic_hpet_scope(struct intel_iommu *iommu) in ir_remove_ioapic_hpet_scope() argument
980 if (ir_hpet[i].iommu == iommu) in ir_remove_ioapic_hpet_scope()
981 ir_hpet[i].iommu = NULL; in ir_remove_ioapic_hpet_scope()
984 if (ir_ioapic[i].iommu == iommu) in ir_remove_ioapic_hpet_scope()
985 ir_ioapic[i].iommu = NULL; in ir_remove_ioapic_hpet_scope()
995 struct intel_iommu *iommu; in parse_ioapics_under_ir() local
999 for_each_iommu(iommu, drhd) { in parse_ioapics_under_ir()
1002 if (!ecap_ir_support(iommu->ecap)) in parse_ioapics_under_ir()
1005 ret = ir_parse_ioapic_hpet_scope(drhd->hdr, iommu); in parse_ioapics_under_ir()
1018 pr_err(FW_BUG "ioapic %d has no mapping iommu, " in parse_ioapics_under_ir()
1046 struct intel_iommu *iommu = NULL; in disable_irq_remapping() local
1051 for_each_iommu(iommu, drhd) { in disable_irq_remapping()
1052 if (!ecap_ir_support(iommu->ecap)) in disable_irq_remapping()
1055 iommu_disable_irq_remapping(iommu); in disable_irq_remapping()
1069 struct intel_iommu *iommu = NULL; in reenable_irq_remapping() local
1071 for_each_iommu(iommu, drhd) in reenable_irq_remapping()
1072 if (iommu->qi) in reenable_irq_remapping()
1073 dmar_reenable_qi(iommu); in reenable_irq_remapping()
1078 for_each_iommu(iommu, drhd) { in reenable_irq_remapping()
1079 if (!ecap_ir_support(iommu->ecap)) in reenable_irq_remapping()
1082 /* Set up interrupt remapping for iommu.*/ in reenable_irq_remapping()
1083 iommu_set_irq_remapping(iommu, eim); in reenable_irq_remapping()
1084 iommu_enable_irq_remapping(iommu); in reenable_irq_remapping()
1358 struct intel_iommu *iommu = domain->host_data; in intel_irq_remapping_alloc() local
1365 if (!info || !iommu) in intel_irq_remapping_alloc()
1388 index = alloc_irte(iommu, &data->irq_2_iommu, nr_irqs); in intel_irq_remapping_alloc()
1466 static int dmar_ir_add(struct dmar_drhd_unit *dmaru, struct intel_iommu *iommu) in dmar_ir_add() argument
1471 if (eim && !ecap_eim_support(iommu->ecap)) { in dmar_ir_add()
1473 iommu->reg_phys, iommu->ecap); in dmar_ir_add()
1477 if (ir_parse_ioapic_hpet_scope(dmaru->hdr, iommu)) { in dmar_ir_add()
1479 iommu->reg_phys); in dmar_ir_add()
1483 /* TODO: check all IOAPICs are covered by IOMMU */ in dmar_ir_add()
1486 ret = intel_setup_irq_remapping(iommu); in dmar_ir_add()
1489 iommu->name); in dmar_ir_add()
1490 intel_teardown_irq_remapping(iommu); in dmar_ir_add()
1491 ir_remove_ioapic_hpet_scope(iommu); in dmar_ir_add()
1493 iommu_enable_irq_remapping(iommu); in dmar_ir_add()
1502 struct intel_iommu *iommu = dmaru->iommu; in dmar_ir_hotplug() local
1506 if (iommu == NULL) in dmar_ir_hotplug()
1508 if (!ecap_ir_support(iommu->ecap)) in dmar_ir_hotplug()
1511 !cap_pi_support(iommu->cap)) in dmar_ir_hotplug()
1515 if (!iommu->ir_table) in dmar_ir_hotplug()
1516 ret = dmar_ir_add(dmaru, iommu); in dmar_ir_hotplug()
1518 if (iommu->ir_table) { in dmar_ir_hotplug()
1519 if (!bitmap_empty(iommu->ir_table->bitmap, in dmar_ir_hotplug()
1523 iommu_disable_irq_remapping(iommu); in dmar_ir_hotplug()
1524 intel_teardown_irq_remapping(iommu); in dmar_ir_hotplug()
1525 ir_remove_ioapic_hpet_scope(iommu); in dmar_ir_hotplug()