Lines Matching +full:iommu +full:- +full:ctx
1 // SPDX-License-Identifier: GPL-2.0-only
3 * omap iommu: tlb and pagetable primitives
5 * Copyright (C) 2008-2010 Nokia Corporation
6 * Copyright (C) 2013-2017 Texas Instruments Incorporated - https://www.ti.com/
12 #include <linux/dma-mapping.h>
18 #include <linux/iommu.h>
19 #include <linux/omap-iommu.h>
31 #include <linux/platform_data/iommu-omap.h>
33 #include "omap-iopgtable.h"
34 #include "omap-iommu.h"
57 * to_omap_domain - Get struct omap_iommu_domain from generic iommu_domain
58 * @dom: generic iommu domain handle
66 * omap_iommu_save_ctx - Save registers for pm off-mode support
82 while (arch_data->iommu_dev) { in omap_iommu_save_ctx()
83 obj = arch_data->iommu_dev; in omap_iommu_save_ctx()
84 p = obj->ctx; in omap_iommu_save_ctx()
87 dev_dbg(obj->dev, "%s\t[%02d] %08x\n", __func__, i, in omap_iommu_save_ctx()
96 * omap_iommu_restore_ctx - Restore registers for pm off-mode support
112 while (arch_data->iommu_dev) { in omap_iommu_restore_ctx()
113 obj = arch_data->iommu_dev; in omap_iommu_restore_ctx()
114 p = obj->ctx; in omap_iommu_restore_ctx()
117 dev_dbg(obj->dev, "%s\t[%02d] %08x\n", __func__, i, in omap_iommu_restore_ctx()
129 if (!obj->syscfg) in dra7_cfg_dspsys_mmu()
132 mask = (1 << (obj->id * DSP_SYS_MMU_CONFIG_EN_SHIFT)); in dra7_cfg_dspsys_mmu()
134 regmap_update_bits(obj->syscfg, DSP_SYS_MMU_CONFIG, mask, val); in dra7_cfg_dspsys_mmu()
159 if (!obj->iopgd || !IS_ALIGNED((unsigned long)obj->iopgd, SZ_16K)) in omap2_iommu_enable()
160 return -EINVAL; in omap2_iommu_enable()
162 pa = virt_to_phys(obj->iopgd); in omap2_iommu_enable()
164 return -EINVAL; in omap2_iommu_enable()
167 dev_info(obj->dev, "%s: version %d.%d\n", obj->name, in omap2_iommu_enable()
174 if (obj->has_bus_err_back) in omap2_iommu_enable()
190 dev_dbg(obj->dev, "%s is shutting down\n", obj->name); in omap2_iommu_disable()
197 ret = pm_runtime_get_sync(obj->dev); in iommu_enable()
199 pm_runtime_put_noidle(obj->dev); in iommu_enable()
206 pm_runtime_put_sync(obj->dev); in iommu_disable()
214 u32 page_size = cr->cam & MMU_CAM_PGSZ_MASK; in iotlb_cr_to_virt()
215 u32 mask = get_cam_va_mask(cr->cam & page_size); in iotlb_cr_to_virt()
217 return cr->cam & mask; in iotlb_cr_to_virt()
224 attr = e->mixed << 5; in get_iopte_attr()
225 attr |= e->endian; in get_iopte_attr()
226 attr |= e->elsz >> 3; in get_iopte_attr()
227 attr <<= (((e->pgsz == MMU_CAM_PGSZ_4K) || in get_iopte_attr()
228 (e->pgsz == MMU_CAM_PGSZ_64K)) ? 0 : 6); in get_iopte_attr()
257 l->base = MMU_LOCK_BASE(val); in iotlb_lock_get()
258 l->vict = MMU_LOCK_VICT(val); in iotlb_lock_get()
265 val = (l->base << MMU_LOCK_BASE_SHIFT); in iotlb_lock_set()
266 val |= (l->vict << MMU_LOCK_VICT_SHIFT); in iotlb_lock_set()
273 cr->cam = iommu_read_reg(obj, MMU_READ_CAM); in iotlb_read_cr()
274 cr->ram = iommu_read_reg(obj, MMU_READ_RAM); in iotlb_read_cr()
279 iommu_write_reg(obj, cr->cam | MMU_CAM_V, MMU_CAM); in iotlb_load_cr()
280 iommu_write_reg(obj, cr->ram, MMU_RAM); in iotlb_load_cr()
286 /* only used in iotlb iteration for-loop */
309 if (e->da & ~(get_cam_va_mask(e->pgsz))) { in iotlb_alloc_cr()
310 dev_err(obj->dev, "%s:\twrong alignment: %08x\n", __func__, in iotlb_alloc_cr()
311 e->da); in iotlb_alloc_cr()
312 return ERR_PTR(-EINVAL); in iotlb_alloc_cr()
317 return ERR_PTR(-ENOMEM); in iotlb_alloc_cr()
319 cr->cam = (e->da & MMU_CAM_VATAG_MASK) | e->prsvd | e->pgsz | e->valid; in iotlb_alloc_cr()
320 cr->ram = e->pa | e->endian | e->elsz | e->mixed; in iotlb_alloc_cr()
326 * load_iotlb_entry - Set an iommu tlb entry
327 * @obj: target iommu
328 * @e: an iommu tlb entry info
336 if (!obj || !obj->nr_tlb_entries || !e) in load_iotlb_entry()
337 return -EINVAL; in load_iotlb_entry()
339 pm_runtime_get_sync(obj->dev); in load_iotlb_entry()
342 if (l.base == obj->nr_tlb_entries) { in load_iotlb_entry()
343 dev_warn(obj->dev, "%s: preserve entries full\n", __func__); in load_iotlb_entry()
344 err = -EBUSY; in load_iotlb_entry()
347 if (!e->prsvd) { in load_iotlb_entry()
351 for_each_iotlb_cr(obj, obj->nr_tlb_entries, i, tmp) in load_iotlb_entry()
355 if (i == obj->nr_tlb_entries) { in load_iotlb_entry()
356 dev_dbg(obj->dev, "%s: full: no entry\n", __func__); in load_iotlb_entry()
357 err = -EBUSY; in load_iotlb_entry()
369 pm_runtime_put_sync(obj->dev); in load_iotlb_entry()
376 if (e->prsvd) in load_iotlb_entry()
379 if (++l.vict == obj->nr_tlb_entries) in load_iotlb_entry()
383 pm_runtime_put_sync(obj->dev); in load_iotlb_entry()
402 * flush_iotlb_page - Clear an iommu tlb entry
403 * @obj: target iommu
404 * @da: iommu device virtual address
406 * Clear an iommu tlb entry which includes 'da' address.
413 pm_runtime_get_sync(obj->dev); in flush_iotlb_page()
415 for_each_iotlb_cr(obj, obj->nr_tlb_entries, i, cr) { in flush_iotlb_page()
426 dev_dbg(obj->dev, "%s: %08x<=%08x(%zx)\n", in flush_iotlb_page()
433 pm_runtime_put_sync(obj->dev); in flush_iotlb_page()
435 if (i == obj->nr_tlb_entries) in flush_iotlb_page()
436 dev_dbg(obj->dev, "%s: no page for %08x\n", __func__, da); in flush_iotlb_page()
440 * flush_iotlb_all - Clear all iommu tlb entries
441 * @obj: target iommu
447 pm_runtime_get_sync(obj->dev); in flush_iotlb_all()
455 pm_runtime_put_sync(obj->dev); in flush_iotlb_all()
473 /* Note: freed iopte's must be clean ready for re-use */ in iopte_free()
477 dma_unmap_single(obj->dev, pt_dma, IOPTE_TABLE_SIZE, in iopte_free()
498 spin_unlock(&obj->page_table_lock); in iopte_alloc()
500 spin_lock(&obj->page_table_lock); in iopte_alloc()
504 return ERR_PTR(-ENOMEM); in iopte_alloc()
506 *pt_dma = dma_map_single(obj->dev, iopte, IOPTE_TABLE_SIZE, in iopte_alloc()
508 if (dma_mapping_error(obj->dev, *pt_dma)) { in iopte_alloc()
509 dev_err(obj->dev, "DMA map error for L2 table\n"); in iopte_alloc()
511 return ERR_PTR(-ENOMEM); in iopte_alloc()
519 dev_err(obj->dev, "DMA translation error for L2 table\n"); in iopte_alloc()
520 dma_unmap_single(obj->dev, *pt_dma, IOPTE_TABLE_SIZE, in iopte_alloc()
523 return ERR_PTR(-ENOMEM); in iopte_alloc()
528 flush_iopte_range(obj->dev, obj->pd_dma, offset, 1); in iopte_alloc()
529 dev_vdbg(obj->dev, "%s: a new pte:%p\n", __func__, iopte); in iopte_alloc()
538 dev_vdbg(obj->dev, in iopte_alloc()
551 dev_err(obj->dev, "%s: %08x:%08x should aligned on %08lx\n", in iopgd_alloc_section()
553 return -EINVAL; in iopgd_alloc_section()
557 flush_iopte_range(obj->dev, obj->pd_dma, offset, 1); in iopgd_alloc_section()
568 dev_err(obj->dev, "%s: %08x:%08x should aligned on %08lx\n", in iopgd_alloc_super()
570 return -EINVAL; in iopgd_alloc_super()
575 flush_iopte_range(obj->dev, obj->pd_dma, offset, 16); in iopgd_alloc_super()
590 flush_iopte_range(obj->dev, pt_dma, offset, 1); in iopte_alloc_page()
592 dev_vdbg(obj->dev, "%s: da:%08x pa:%08x pte:%p *pte:%08x\n", in iopte_alloc_page()
607 dev_err(obj->dev, "%s: %08x:%08x should aligned on %08lx\n", in iopte_alloc_large()
609 return -EINVAL; in iopte_alloc_large()
617 flush_iopte_range(obj->dev, pt_dma, offset, 16); in iopte_alloc_large()
629 return -EINVAL; in iopgtable_store_entry_core()
631 switch (e->pgsz) { in iopgtable_store_entry_core()
650 return -EINVAL; in iopgtable_store_entry_core()
654 spin_lock(&obj->page_table_lock); in iopgtable_store_entry_core()
655 err = fn(obj, e->da, e->pa, prot); in iopgtable_store_entry_core()
656 spin_unlock(&obj->page_table_lock); in iopgtable_store_entry_core()
662 * omap_iopgtable_store_entry - Make an iommu pte entry
663 * @obj: target iommu
664 * @e: an iommu tlb entry info
671 flush_iotlb_page(obj, e->da); in omap_iopgtable_store_entry()
679 * iopgtable_lookup_entry - Lookup an iommu pte entry
680 * @obj: target iommu
681 * @da: iommu device virtual address
682 * @ppgd: iommu pgd entry pointer to be returned
683 * @ppte: iommu pte entry pointer to be returned
726 flush_iopte_range(obj->dev, pt_dma, pt_offset, nent); in iopgtable_clear_entry_core()
748 flush_iopte_range(obj->dev, obj->pd_dma, pd_offset, nent); in iopgtable_clear_entry_core()
754 * iopgtable_clear_entry - Remove an iommu pte entry
755 * @obj: target iommu
756 * @da: iommu device virtual address
762 spin_lock(&obj->page_table_lock); in iopgtable_clear_entry()
767 spin_unlock(&obj->page_table_lock); in iopgtable_clear_entry()
777 spin_lock(&obj->page_table_lock); in iopgtable_clear_entry_all()
794 flush_iopte_range(obj->dev, obj->pd_dma, offset, 1); in iopgtable_clear_entry_all()
799 spin_unlock(&obj->page_table_lock); in iopgtable_clear_entry_all()
803 * Device IOMMU generic operations
810 struct iommu_domain *domain = obj->domain; in iommu_fault_handler()
813 if (!omap_domain->dev) in iommu_fault_handler()
821 if (!report_iommu_fault(domain, obj->dev, da, 0)) in iommu_fault_handler()
829 dev_err(obj->dev, "%s: errs:0x%08x da:0x%08x pgd:0x%p *pgd:px%08x\n", in iommu_fault_handler()
830 obj->name, errs, da, iopgd, *iopgd); in iommu_fault_handler()
836 dev_err(obj->dev, "%s: errs:0x%08x da:0x%08x pgd:0x%p *pgd:0x%08x pte:0x%p *pte:0x%08x\n", in iommu_fault_handler()
837 obj->name, errs, da, iopgd, *iopgd, iopte, *iopte); in iommu_fault_handler()
843 * omap_iommu_attach() - attach iommu device to an iommu domain
844 * @obj: target omap iommu device
851 spin_lock(&obj->iommu_lock); in omap_iommu_attach()
853 obj->pd_dma = dma_map_single(obj->dev, iopgd, IOPGD_TABLE_SIZE, in omap_iommu_attach()
855 if (dma_mapping_error(obj->dev, obj->pd_dma)) { in omap_iommu_attach()
856 dev_err(obj->dev, "DMA map error for L1 table\n"); in omap_iommu_attach()
857 err = -ENOMEM; in omap_iommu_attach()
861 obj->iopgd = iopgd; in omap_iommu_attach()
867 spin_unlock(&obj->iommu_lock); in omap_iommu_attach()
869 dev_dbg(obj->dev, "%s: %s\n", __func__, obj->name); in omap_iommu_attach()
874 spin_unlock(&obj->iommu_lock); in omap_iommu_attach()
880 * omap_iommu_detach - release iommu device
881 * @obj: target iommu
888 spin_lock(&obj->iommu_lock); in omap_iommu_detach()
890 dma_unmap_single(obj->dev, obj->pd_dma, IOPGD_TABLE_SIZE, in omap_iommu_detach()
892 obj->pd_dma = 0; in omap_iommu_detach()
893 obj->iopgd = NULL; in omap_iommu_detach()
896 spin_unlock(&obj->iommu_lock); in omap_iommu_detach()
898 dev_dbg(obj->dev, "%s: %s\n", __func__, obj->name); in omap_iommu_detach()
910 obj->num_cr_ctx = lock.base; in omap_iommu_save_tlb_entries()
911 if (!obj->num_cr_ctx) in omap_iommu_save_tlb_entries()
914 tmp = obj->cr_ctx; in omap_iommu_save_tlb_entries()
915 for_each_iotlb_cr(obj, obj->num_cr_ctx, i, cr) in omap_iommu_save_tlb_entries()
926 if (!obj->num_cr_ctx) in omap_iommu_restore_tlb_entries()
930 tmp = obj->cr_ctx; in omap_iommu_restore_tlb_entries()
931 for (i = 0; i < obj->num_cr_ctx; i++, tmp++) { in omap_iommu_restore_tlb_entries()
936 l.base = obj->num_cr_ctx; in omap_iommu_restore_tlb_entries()
942 * omap_iommu_domain_deactivate - deactivate attached iommu devices
943 * @domain: iommu domain attached to the target iommu device
945 * This API allows the client devices of IOMMU devices to suspend
953 struct omap_iommu_device *iommu; in omap_iommu_domain_deactivate() local
957 if (!omap_domain->dev) in omap_iommu_domain_deactivate()
960 iommu = omap_domain->iommus; in omap_iommu_domain_deactivate()
961 iommu += (omap_domain->num_iommus - 1); in omap_iommu_domain_deactivate()
962 for (i = 0; i < omap_domain->num_iommus; i++, iommu--) { in omap_iommu_domain_deactivate()
963 oiommu = iommu->iommu_dev; in omap_iommu_domain_deactivate()
964 pm_runtime_put_sync(oiommu->dev); in omap_iommu_domain_deactivate()
972 * omap_iommu_domain_activate - activate attached iommu devices
973 * @domain: iommu domain attached to the target iommu device
975 * This API allows the client devices of IOMMU devices to resume the
982 struct omap_iommu_device *iommu; in omap_iommu_domain_activate() local
986 if (!omap_domain->dev) in omap_iommu_domain_activate()
989 iommu = omap_domain->iommus; in omap_iommu_domain_activate()
990 for (i = 0; i < omap_domain->num_iommus; i++, iommu++) { in omap_iommu_domain_activate()
991 oiommu = iommu->iommu_dev; in omap_iommu_domain_activate()
992 pm_runtime_get_sync(oiommu->dev); in omap_iommu_domain_activate()
1000 * omap_iommu_runtime_suspend - disable an iommu device
1001 * @dev: iommu device
1004 * IOMMU device, either during final detachment from a client
1006 * includes programming all the appropriate IOMMU registers, and
1019 if (obj->domain && obj->iopgd) in omap_iommu_runtime_suspend()
1024 if (pdata && pdata->device_idle) in omap_iommu_runtime_suspend()
1025 pdata->device_idle(pdev); in omap_iommu_runtime_suspend()
1027 if (pdata && pdata->assert_reset) in omap_iommu_runtime_suspend()
1028 pdata->assert_reset(pdev, pdata->reset_name); in omap_iommu_runtime_suspend()
1030 if (pdata && pdata->set_pwrdm_constraint) { in omap_iommu_runtime_suspend()
1031 ret = pdata->set_pwrdm_constraint(pdev, false, &obj->pwrst); in omap_iommu_runtime_suspend()
1033 dev_warn(obj->dev, "pwrdm_constraint failed to be reset, status = %d\n", in omap_iommu_runtime_suspend()
1042 * omap_iommu_runtime_resume - enable an iommu device
1043 * @dev: iommu device
1046 * IOMMU device, either during initial attachment to a client
1048 * includes programming all the appropriate IOMMU registers, and
1060 if (pdata && pdata->set_pwrdm_constraint) { in omap_iommu_runtime_resume()
1061 ret = pdata->set_pwrdm_constraint(pdev, true, &obj->pwrst); in omap_iommu_runtime_resume()
1063 dev_warn(obj->dev, "pwrdm_constraint failed to be set, status = %d\n", in omap_iommu_runtime_resume()
1068 if (pdata && pdata->deassert_reset) { in omap_iommu_runtime_resume()
1069 ret = pdata->deassert_reset(pdev, pdata->reset_name); in omap_iommu_runtime_resume()
1076 if (pdata && pdata->device_enable) in omap_iommu_runtime_resume()
1077 pdata->device_enable(pdev); in omap_iommu_runtime_resume()
1080 if (obj->domain) in omap_iommu_runtime_resume()
1089 * omap_iommu_suspend_prepare - prepare() dev_pm_ops implementation
1090 * @dev: iommu device
1092 * This function performs the necessary checks to determine if the IOMMU
1107 struct device_node *np = pdev->dev.of_node; in omap_iommu_can_register()
1109 if (!of_device_is_compatible(np, "ti,dra7-dsp-iommu")) in omap_iommu_can_register()
1113 * restrict IOMMU core registration only for processor-port MDMA MMUs in omap_iommu_can_register()
1116 if ((!strcmp(dev_name(&pdev->dev), "40d01000.mmu")) || in omap_iommu_can_register()
1117 (!strcmp(dev_name(&pdev->dev), "41501000.mmu"))) in omap_iommu_can_register()
1126 struct device_node *np = pdev->dev.of_node; in omap_iommu_dra7_get_dsp_system_cfg()
1129 if (!of_device_is_compatible(np, "ti,dra7-dsp-iommu")) in omap_iommu_dra7_get_dsp_system_cfg()
1132 if (!of_property_read_bool(np, "ti,syscon-mmuconfig")) { in omap_iommu_dra7_get_dsp_system_cfg()
1133 dev_err(&pdev->dev, "ti,syscon-mmuconfig property is missing\n"); in omap_iommu_dra7_get_dsp_system_cfg()
1134 return -EINVAL; in omap_iommu_dra7_get_dsp_system_cfg()
1137 obj->syscfg = in omap_iommu_dra7_get_dsp_system_cfg()
1138 syscon_regmap_lookup_by_phandle(np, "ti,syscon-mmuconfig"); in omap_iommu_dra7_get_dsp_system_cfg()
1139 if (IS_ERR(obj->syscfg)) { in omap_iommu_dra7_get_dsp_system_cfg()
1140 /* can fail with -EPROBE_DEFER */ in omap_iommu_dra7_get_dsp_system_cfg()
1141 ret = PTR_ERR(obj->syscfg); in omap_iommu_dra7_get_dsp_system_cfg()
1145 if (of_property_read_u32_index(np, "ti,syscon-mmuconfig", 1, in omap_iommu_dra7_get_dsp_system_cfg()
1146 &obj->id)) { in omap_iommu_dra7_get_dsp_system_cfg()
1147 dev_err(&pdev->dev, "couldn't get the IOMMU instance id within subsystem\n"); in omap_iommu_dra7_get_dsp_system_cfg()
1148 return -EINVAL; in omap_iommu_dra7_get_dsp_system_cfg()
1151 if (obj->id != 0 && obj->id != 1) { in omap_iommu_dra7_get_dsp_system_cfg()
1152 dev_err(&pdev->dev, "invalid IOMMU instance id\n"); in omap_iommu_dra7_get_dsp_system_cfg()
1153 return -EINVAL; in omap_iommu_dra7_get_dsp_system_cfg()
1160 * OMAP Device MMU(IOMMU) detection
1164 int err = -ENODEV; in omap_iommu_probe()
1168 struct device_node *of = pdev->dev.of_node; in omap_iommu_probe()
1171 pr_err("%s: only DT-based devices are supported\n", __func__); in omap_iommu_probe()
1172 return -ENODEV; in omap_iommu_probe()
1175 obj = devm_kzalloc(&pdev->dev, sizeof(*obj) + MMU_REG_SIZE, GFP_KERNEL); in omap_iommu_probe()
1177 return -ENOMEM; in omap_iommu_probe()
1180 * self-manage the ordering dependencies between omap_device_enable/idle in omap_iommu_probe()
1183 if (pdev->dev.pm_domain) { in omap_iommu_probe()
1184 dev_dbg(&pdev->dev, "device pm_domain is being reset\n"); in omap_iommu_probe()
1185 pdev->dev.pm_domain = NULL; in omap_iommu_probe()
1188 obj->name = dev_name(&pdev->dev); in omap_iommu_probe()
1189 obj->nr_tlb_entries = 32; in omap_iommu_probe()
1190 err = of_property_read_u32(of, "ti,#tlb-entries", &obj->nr_tlb_entries); in omap_iommu_probe()
1191 if (err && err != -EINVAL) in omap_iommu_probe()
1193 if (obj->nr_tlb_entries != 32 && obj->nr_tlb_entries != 8) in omap_iommu_probe()
1194 return -EINVAL; in omap_iommu_probe()
1195 if (of_find_property(of, "ti,iommu-bus-err-back", NULL)) in omap_iommu_probe()
1196 obj->has_bus_err_back = MMU_GP_REG_BUS_ERR_BACK_EN; in omap_iommu_probe()
1198 obj->dev = &pdev->dev; in omap_iommu_probe()
1199 obj->ctx = (void *)obj + sizeof(*obj); in omap_iommu_probe()
1200 obj->cr_ctx = devm_kzalloc(&pdev->dev, in omap_iommu_probe()
1201 sizeof(*obj->cr_ctx) * obj->nr_tlb_entries, in omap_iommu_probe()
1203 if (!obj->cr_ctx) in omap_iommu_probe()
1204 return -ENOMEM; in omap_iommu_probe()
1206 spin_lock_init(&obj->iommu_lock); in omap_iommu_probe()
1207 spin_lock_init(&obj->page_table_lock); in omap_iommu_probe()
1210 obj->regbase = devm_ioremap_resource(obj->dev, res); in omap_iommu_probe()
1211 if (IS_ERR(obj->regbase)) in omap_iommu_probe()
1212 return PTR_ERR(obj->regbase); in omap_iommu_probe()
1220 return -ENODEV; in omap_iommu_probe()
1222 err = devm_request_irq(obj->dev, irq, iommu_fault_handler, IRQF_SHARED, in omap_iommu_probe()
1223 dev_name(obj->dev), obj); in omap_iommu_probe()
1229 obj->group = iommu_group_alloc(); in omap_iommu_probe()
1230 if (IS_ERR(obj->group)) in omap_iommu_probe()
1231 return PTR_ERR(obj->group); in omap_iommu_probe()
1233 err = iommu_device_sysfs_add(&obj->iommu, obj->dev, NULL, in omap_iommu_probe()
1234 obj->name); in omap_iommu_probe()
1238 iommu_device_set_ops(&obj->iommu, &omap_iommu_ops); in omap_iommu_probe()
1239 iommu_device_set_fwnode(&obj->iommu, &of->fwnode); in omap_iommu_probe()
1241 err = iommu_device_register(&obj->iommu); in omap_iommu_probe()
1246 pm_runtime_enable(obj->dev); in omap_iommu_probe()
1250 dev_info(&pdev->dev, "%s registered\n", obj->name); in omap_iommu_probe()
1252 /* Re-probe bus to probe device attached to this IOMMU */ in omap_iommu_probe()
1258 iommu_device_sysfs_remove(&obj->iommu); in omap_iommu_probe()
1260 iommu_group_put(obj->group); in omap_iommu_probe()
1268 if (obj->group) { in omap_iommu_remove()
1269 iommu_group_put(obj->group); in omap_iommu_remove()
1270 obj->group = NULL; in omap_iommu_remove()
1272 iommu_device_sysfs_remove(&obj->iommu); in omap_iommu_remove()
1273 iommu_device_unregister(&obj->iommu); in omap_iommu_remove()
1278 pm_runtime_disable(obj->dev); in omap_iommu_remove()
1280 dev_info(&pdev->dev, "%s removed\n", obj->name); in omap_iommu_remove()
1293 { .compatible = "ti,omap2-iommu" },
1294 { .compatible = "ti,omap4-iommu" },
1295 { .compatible = "ti,dra7-iommu" },
1296 { .compatible = "ti,dra7-dsp-iommu" },
1304 .name = "omap-iommu",
1314 e->da = da; in iotlb_init_entry()
1315 e->pa = pa; in iotlb_init_entry()
1316 e->valid = MMU_CAM_V; in iotlb_init_entry()
1317 e->pgsz = pgsz; in iotlb_init_entry()
1318 e->endian = MMU_RAM_ENDIAN_LITTLE; in iotlb_init_entry()
1319 e->elsz = MMU_RAM_ELSZ_8; in iotlb_init_entry()
1320 e->mixed = 0; in iotlb_init_entry()
1322 return iopgsz_to_bytes(e->pgsz); in iotlb_init_entry()
1329 struct device *dev = omap_domain->dev; in omap_iommu_map()
1330 struct omap_iommu_device *iommu; in omap_iommu_map() local
1334 u32 ret = -EINVAL; in omap_iommu_map()
1340 return -EINVAL; in omap_iommu_map()
1347 iommu = omap_domain->iommus; in omap_iommu_map()
1348 for (i = 0; i < omap_domain->num_iommus; i++, iommu++) { in omap_iommu_map()
1349 oiommu = iommu->iommu_dev; in omap_iommu_map()
1359 while (i--) { in omap_iommu_map()
1360 iommu--; in omap_iommu_map()
1361 oiommu = iommu->iommu_dev; in omap_iommu_map()
1373 struct device *dev = omap_domain->dev; in omap_iommu_unmap()
1374 struct omap_iommu_device *iommu; in omap_iommu_unmap() local
1382 iommu = omap_domain->iommus; in omap_iommu_unmap()
1383 for (i = 0; i < omap_domain->num_iommus; i++, iommu++) { in omap_iommu_unmap()
1384 oiommu = iommu->iommu_dev; in omap_iommu_unmap()
1391 * simplify return - we are only checking if any of the iommus in omap_iommu_unmap()
1404 while (arch_data->iommu_dev) { in omap_iommu_count()
1416 struct omap_iommu_device *iommu; in omap_iommu_attach_init() local
1419 odomain->num_iommus = omap_iommu_count(dev); in omap_iommu_attach_init()
1420 if (!odomain->num_iommus) in omap_iommu_attach_init()
1421 return -EINVAL; in omap_iommu_attach_init()
1423 odomain->iommus = kcalloc(odomain->num_iommus, sizeof(*iommu), in omap_iommu_attach_init()
1425 if (!odomain->iommus) in omap_iommu_attach_init()
1426 return -ENOMEM; in omap_iommu_attach_init()
1428 iommu = odomain->iommus; in omap_iommu_attach_init()
1429 for (i = 0; i < odomain->num_iommus; i++, iommu++) { in omap_iommu_attach_init()
1430 iommu->pgtable = kzalloc(IOPGD_TABLE_SIZE, GFP_ATOMIC); in omap_iommu_attach_init()
1431 if (!iommu->pgtable) in omap_iommu_attach_init()
1432 return -ENOMEM; in omap_iommu_attach_init()
1438 if (WARN_ON(!IS_ALIGNED((long)iommu->pgtable, in omap_iommu_attach_init()
1440 return -EINVAL; in omap_iommu_attach_init()
1449 struct omap_iommu_device *iommu = odomain->iommus; in omap_iommu_detach_fini() local
1451 for (i = 0; iommu && i < odomain->num_iommus; i++, iommu++) in omap_iommu_detach_fini()
1452 kfree(iommu->pgtable); in omap_iommu_detach_fini()
1454 kfree(odomain->iommus); in omap_iommu_detach_fini()
1455 odomain->num_iommus = 0; in omap_iommu_detach_fini()
1456 odomain->iommus = NULL; in omap_iommu_detach_fini()
1464 struct omap_iommu_device *iommu; in omap_iommu_attach_dev() local
1469 if (!arch_data || !arch_data->iommu_dev) { in omap_iommu_attach_dev()
1470 dev_err(dev, "device doesn't have an associated iommu\n"); in omap_iommu_attach_dev()
1471 return -EINVAL; in omap_iommu_attach_dev()
1474 spin_lock(&omap_domain->lock); in omap_iommu_attach_dev()
1477 if (omap_domain->dev) { in omap_iommu_attach_dev()
1478 dev_err(dev, "iommu domain is already attached\n"); in omap_iommu_attach_dev()
1479 ret = -EBUSY; in omap_iommu_attach_dev()
1485 dev_err(dev, "failed to allocate required iommu data %d\n", in omap_iommu_attach_dev()
1490 iommu = omap_domain->iommus; in omap_iommu_attach_dev()
1491 for (i = 0; i < omap_domain->num_iommus; i++, iommu++, arch_data++) { in omap_iommu_attach_dev()
1492 /* configure and enable the omap iommu */ in omap_iommu_attach_dev()
1493 oiommu = arch_data->iommu_dev; in omap_iommu_attach_dev()
1494 ret = omap_iommu_attach(oiommu, iommu->pgtable); in omap_iommu_attach_dev()
1496 dev_err(dev, "can't get omap iommu: %d\n", ret); in omap_iommu_attach_dev()
1500 oiommu->domain = domain; in omap_iommu_attach_dev()
1501 iommu->iommu_dev = oiommu; in omap_iommu_attach_dev()
1504 omap_domain->dev = dev; in omap_iommu_attach_dev()
1509 while (i--) { in omap_iommu_attach_dev()
1510 iommu--; in omap_iommu_attach_dev()
1511 arch_data--; in omap_iommu_attach_dev()
1512 oiommu = iommu->iommu_dev; in omap_iommu_attach_dev()
1514 iommu->iommu_dev = NULL; in omap_iommu_attach_dev()
1515 oiommu->domain = NULL; in omap_iommu_attach_dev()
1520 spin_unlock(&omap_domain->lock); in omap_iommu_attach_dev()
1528 struct omap_iommu_device *iommu = omap_domain->iommus; in _omap_iommu_detach_dev() local
1532 if (!omap_domain->dev) { in _omap_iommu_detach_dev()
1538 if (omap_domain->dev != dev) { in _omap_iommu_detach_dev()
1544 * cleanup in the reverse order of attachment - this addresses in _omap_iommu_detach_dev()
1547 iommu += (omap_domain->num_iommus - 1); in _omap_iommu_detach_dev()
1548 arch_data += (omap_domain->num_iommus - 1); in _omap_iommu_detach_dev()
1549 for (i = 0; i < omap_domain->num_iommus; i++, iommu--, arch_data--) { in _omap_iommu_detach_dev()
1550 oiommu = iommu->iommu_dev; in _omap_iommu_detach_dev()
1554 iommu->iommu_dev = NULL; in _omap_iommu_detach_dev()
1555 oiommu->domain = NULL; in _omap_iommu_detach_dev()
1560 omap_domain->dev = NULL; in _omap_iommu_detach_dev()
1568 spin_lock(&omap_domain->lock); in omap_iommu_detach_dev()
1570 spin_unlock(&omap_domain->lock); in omap_iommu_detach_dev()
1584 spin_lock_init(&omap_domain->lock); in omap_iommu_domain_alloc()
1586 omap_domain->domain.geometry.aperture_start = 0; in omap_iommu_domain_alloc()
1587 omap_domain->domain.geometry.aperture_end = (1ULL << 32) - 1; in omap_iommu_domain_alloc()
1588 omap_domain->domain.geometry.force_aperture = true; in omap_iommu_domain_alloc()
1590 return &omap_domain->domain; in omap_iommu_domain_alloc()
1598 * An iommu device is still attached in omap_iommu_domain_free()
1601 if (omap_domain->dev) in omap_iommu_domain_free()
1602 _omap_iommu_detach_dev(omap_domain, omap_domain->dev); in omap_iommu_domain_free()
1611 struct omap_iommu_device *iommu = omap_domain->iommus; in omap_iommu_iova_to_phys() local
1612 struct omap_iommu *oiommu = iommu->iommu_dev; in omap_iommu_iova_to_phys()
1613 struct device *dev = oiommu->dev; in omap_iommu_iova_to_phys()
1619 * so perform the lookup using just the first iommu in omap_iommu_iova_to_phys()
1653 * Allocate the per-device iommu structure for DT-based devices. in omap_iommu_probe_device()
1655 * TODO: Simplify this when removing non-DT support completely from the in omap_iommu_probe_device()
1656 * IOMMU users. in omap_iommu_probe_device()
1658 if (!dev->of_node) in omap_iommu_probe_device()
1659 return ERR_PTR(-ENODEV); in omap_iommu_probe_device()
1662 * retrieve the count of IOMMU nodes using phandle size as element size in omap_iommu_probe_device()
1663 * since #iommu-cells = 0 for OMAP in omap_iommu_probe_device()
1665 num_iommus = of_property_count_elems_of_size(dev->of_node, "iommus", in omap_iommu_probe_device()
1668 return ERR_PTR(-ENODEV); in omap_iommu_probe_device()
1672 return ERR_PTR(-ENOMEM); in omap_iommu_probe_device()
1675 np = of_parse_phandle(dev->of_node, "iommus", i); in omap_iommu_probe_device()
1678 return ERR_PTR(-EINVAL); in omap_iommu_probe_device()
1685 return ERR_PTR(-ENODEV); in omap_iommu_probe_device()
1692 return ERR_PTR(-EINVAL); in omap_iommu_probe_device()
1695 tmp->iommu_dev = oiommu; in omap_iommu_probe_device()
1696 tmp->dev = &pdev->dev; in omap_iommu_probe_device()
1704 * use the first IOMMU alone for the sysfs device linking. in omap_iommu_probe_device()
1708 oiommu = arch_data->iommu_dev; in omap_iommu_probe_device()
1710 return &oiommu->iommu; in omap_iommu_probe_device()
1717 if (!dev->of_node || !arch_data) in omap_iommu_release_device()
1728 struct iommu_group *group = ERR_PTR(-EINVAL); in omap_iommu_device_group()
1731 return ERR_PTR(-ENODEV); in omap_iommu_device_group()
1733 if (arch_data->iommu_dev) in omap_iommu_device_group()
1734 group = iommu_group_ref_get(arch_data->iommu_dev->group); in omap_iommu_device_group()
1770 return -ENOMEM; in omap_iommu_init()