191     const struct iommu_ops *ops = dev->bus->iommu_ops;  in iommu_probe_device_ext()  local
196     if (!ops) {  in iommu_probe_device_ext()
204     if (!try_module_get(ops->owner)) {  in iommu_probe_device_ext()
209     iommu_dev = ops->probe_device(dev);  in iommu_probe_device_ext()
233     ops->release_device(dev);  in iommu_probe_device_ext()
236     module_put(ops->owner);  in iommu_probe_device_ext()
246     const struct iommu_ops *ops = dev->bus->iommu_ops;  in iommu_probe_device()  local
280     if (ops->probe_finalize) {  in iommu_probe_device()
281         ops->probe_finalize(dev);  in iommu_probe_device()
295     const struct iommu_ops *ops = dev->bus->iommu_ops;  in iommu_release_device()  local
303     ops->release_device(dev);  in iommu_release_device()
306     module_put(ops->owner);  in iommu_release_device()
747         if (domain->ops->apply_resv_region) {  in iommu_create_device_direct_mappings()
748             domain->ops->apply_resv_region(dev, domain, entry);  in iommu_create_device_direct_mappings()
783     if (domain->ops->is_attach_deferred) {  in iommu_is_attach_deferred()
784         return domain->ops->is_attach_deferred(domain, dev);  in iommu_is_attach_deferred()
1209     if (!domain || !domain->ops->page_response) {  in iommu_page_response()
1255         ret = domain->ops->page_response(dev, evt, msg);  in iommu_page_response()
1491     const struct iommu_ops *ops = dev->bus->iommu_ops;  in iommu_get_def_domain_type()  local
1494     if (ops->def_domain_type) {  in iommu_get_def_domain_type()
1495         type = ops->def_domain_type(dev);  in iommu_get_def_domain_type()
1557     const struct iommu_ops *ops = dev->bus->iommu_ops;  in iommu_group_get_for_dev()  local
1566     if (!ops) {  in iommu_group_get_for_dev()
1570     group = ops->device_group(dev);  in iommu_group_get_for_dev()
1686     const struct iommu_ops *ops = dev->bus->iommu_ops;  in probe_get_default_domain_type()  local
1690     if (ops->def_domain_type) {  in probe_get_default_domain_type()
1691         type = ops->def_domain_type(dev);  in probe_get_default_domain_type()
1749     if (domain->ops->probe_finalize) {  in iommu_group_do_probe_finalize()
1750         domain->ops->probe_finalize(dev);  in iommu_group_do_probe_finalize()
1822 static int iommu_bus_init(struct bus_type *bus, const struct iommu_ops *ops)  in iommu_bus_init()  argument
1870 int bus_set_iommu(struct bus_type *bus, const struct iommu_ops *ops)  in bus_set_iommu()  argument
1874     if (ops == NULL) {  in bus_set_iommu()
1883     bus->iommu_ops = ops;  in bus_set_iommu()
1886     err = iommu_bus_init(bus, ops);  in bus_set_iommu()
1945     domain->ops = bus->iommu_ops;  in iommu_domain_alloc_ext()
1961     domain->ops->domain_free(domain);  in iommu_domain_free()
1969     if (unlikely(domain->ops->attach_dev == NULL)) {  in iommu_attach_device_ext()
1973     ret = domain->ops->attach_dev(domain, dev);  in iommu_attach_device_ext()
2077     if (unlikely(!domain->ops->cache_invalidate)) {  in iommu_uapi_cache_invalidate()
2125     return domain->ops->cache_invalidate(domain, dev, &inv_info);  in iommu_uapi_cache_invalidate()
2197     if (unlikely(!domain->ops->sva_bind_gpasid)) {  in iommu_uapi_sva_bind_gpasid()
2206     return domain->ops->sva_bind_gpasid(domain, dev, &data);  in iommu_uapi_sva_bind_gpasid()
2212     if (unlikely(!domain->ops->sva_unbind_gpasid)) {  in iommu_sva_unbind_gpasid()
2216     return domain->ops->sva_unbind_gpasid(dev, pasid);  in iommu_sva_unbind_gpasid()
2225     if (unlikely(!domain->ops->sva_bind_gpasid)) {  in iommu_uapi_sva_unbind_gpasid()
2244     if (unlikely(domain->ops->detach_dev == NULL)) {  in __iommu_detach_device()
2248     domain->ops->detach_dev(domain, dev);  in __iommu_detach_device()
2390     if (unlikely(domain->ops->iova_to_phys == NULL)) {  in iommu_iova_to_phys()
2394     return domain->ops->iova_to_phys(domain, iova);  in iommu_iova_to_phys()
2451     const struct iommu_ops *ops = domain->ops;  in iommu_map_pages_ext()  local
2459     if (ops->map_pages) {  in iommu_map_pages_ext()
2460         ret = ops->map_pages(domain, iova, paddr, pgsize, count, prot, gfp, mapped);  in iommu_map_pages_ext()
2462         ret = ops->map(domain, iova, paddr, pgsize, prot, gfp);  in iommu_map_pages_ext()
2472     const struct iommu_ops *ops = domain->ops;  in iommu_map_ext()  local
2479     if (unlikely(!(ops->map || ops->map_pages) || domain->pgsize_bitmap == 0UL)) {  in iommu_map_ext()
2532     const struct iommu_ops *ops = domain->ops;  in _iommu_map()  local
2536     if (ret == 0 && ops->iotlb_sync_map) {  in _iommu_map()
2537         ops->iotlb_sync_map(domain, iova, size);  in _iommu_map()
2559     const struct iommu_ops *ops = domain->ops;  in iommu_unmap_pages_ext()  local
2563     return ops->unmap_pages ? ops->unmap_pages(domain, iova, pgsize, count, iotlb_gather)  in iommu_unmap_pages_ext()
2564                             : ops->unmap(domain, iova, pgsize, iotlb_gather);  in iommu_unmap_pages_ext()
2570     const struct iommu_ops *ops = domain->ops;  in iommu_unmap_ext()  local
2575     if (unlikely(!(ops->unmap || ops->unmap_pages) || domain->pgsize_bitmap == 0UL)) {  in iommu_unmap_ext()
2639     const struct iommu_ops *ops = domain->ops;  in iommu_map_sg_ext()  local
2645     if (ops->map_sg) {  in iommu_map_sg_ext()
2646         ret = ops->map_sg(domain, iova, sg, nents, prot, gfp, &mapped);  in iommu_map_sg_ext()
2648         if (ops->iotlb_sync_map) {  in iommu_map_sg_ext()
2649             ops->iotlb_sync_map(domain, iova, mapped);  in iommu_map_sg_ext()
2683     if (ops->iotlb_sync_map) {  in iommu_map_sg_ext()
2684         ops->iotlb_sync_map(domain, iova, mapped);  in iommu_map_sg_ext()
2688     if (domain->ops->flush_iotlb_all && (prot & IOMMU_TLB_SHOT_ENTIRE)) {  in iommu_map_sg_ext()
2689         domain->ops->flush_iotlb_all(domain);  in iommu_map_sg_ext()
2719     if (unlikely(domain->ops->domain_window_enable == NULL)) {  in iommu_domain_window_enable()
2723     return domain->ops->domain_window_enable(domain, wnd_nr, paddr, size, prot);  in iommu_domain_window_enable()
2729     if (unlikely(domain->ops->domain_window_disable == NULL)) {  in iommu_domain_window_disable()
2733     return domain->ops->domain_window_disable(domain, wnd_nr);  in iommu_domain_window_disable()
2806             if (!domain->ops->domain_get_attr) {  in iommu_domain_get_attr()
2810             ret = domain->ops->domain_get_attr(domain, attr, data);  in iommu_domain_get_attr()
2821     if (domain->ops->domain_set_attr == NULL) {  in iommu_domain_set_attr()
2825     ret = domain->ops->domain_set_attr(domain, attr, data);  in iommu_domain_set_attr()
2832     const struct iommu_ops *ops = dev->bus->iommu_ops;  in iommu_get_resv_regions()  local
2834     if (ops && ops->get_resv_regions) {  in iommu_get_resv_regions()
2835         ops->get_resv_regions(dev, list);  in iommu_get_resv_regions()
2841     const struct iommu_ops *ops = dev->bus->iommu_ops;  in iommu_put_resv_regions()  local
2843     if (ops && ops->put_resv_regions) {  in iommu_put_resv_regions()
2844         ops->put_resv_regions(dev, list);  in iommu_put_resv_regions()
2910     const struct iommu_ops *ops = NULL;  in iommu_ops_from_fwnode()  local
2917             ops = iommu->ops;  in iommu_ops_from_fwnode()
2922     return ops;  in iommu_ops_from_fwnode()
2925 …mu_fwspec_init(struct device *dev, struct fwnode_handle *iommu_fwnode, const struct iommu_ops *ops)  in iommu_fwspec_init()  argument
2930         return ops == fwspec->ops ? 0 : -EINVAL;  in iommu_fwspec_init()
2945     fwspec->ops = ops;  in iommu_fwspec_init()
2996     const struct iommu_ops *ops = dev->bus->iommu_ops;  in iommu_dev_has_feature()  local
2998     if (ops && ops->dev_has_feat) {  in iommu_dev_has_feature()
2999         return ops->dev_has_feat(dev, feat);  in iommu_dev_has_feature()
3009         const struct iommu_ops *ops = dev->iommu->iommu_dev->ops;  in iommu_dev_enable_feature()  local
3011         if (ops->dev_enable_feat) {  in iommu_dev_enable_feature()
3012             return ops->dev_enable_feat(dev, feat);  in iommu_dev_enable_feature()
3028         const struct iommu_ops *ops = dev->iommu->iommu_dev->ops;  in iommu_dev_disable_feature()  local
3030         if (ops->dev_disable_feat) {  in iommu_dev_disable_feature()
3031             return ops->dev_disable_feat(dev, feat);  in iommu_dev_disable_feature()
3042         const struct iommu_ops *ops = dev->iommu->iommu_dev->ops;  in iommu_dev_feature_enabled()  local
3044         if (ops->dev_feat_enabled) {  in iommu_dev_feature_enabled()
3045             return ops->dev_feat_enabled(dev, feat);  in iommu_dev_feature_enabled()
3067     if (domain->ops->aux_attach_dev) {  in iommu_aux_attach_device()
3068         ret = domain->ops->aux_attach_dev(domain, dev);  in iommu_aux_attach_device()
3081     if (domain->ops->aux_detach_dev) {  in iommu_aux_detach_device()
3082         domain->ops->aux_detach_dev(domain, dev);  in iommu_aux_detach_device()
3092     if (domain->ops->aux_get_pasid) {  in iommu_aux_get_pasid()
3093         ret = domain->ops->aux_get_pasid(domain, dev);  in iommu_aux_get_pasid()
3119     const struct iommu_ops *ops = dev->bus->iommu_ops;  in iommu_sva_bind_device()  local
3121     if (!ops || !ops->sva_bind) {  in iommu_sva_bind_device()
3143     handle = ops->sva_bind(dev, mm, drvdata);  in iommu_sva_bind_device()
3167     const struct iommu_ops *ops = dev->bus->iommu_ops;  in iommu_sva_unbind_device()  local
3169     if (!ops || !ops->sva_unbind) {  in iommu_sva_unbind_device()
3179     ops->sva_unbind(handle);  in iommu_sva_unbind_device()
3188     const struct iommu_ops *ops = handle->dev->bus->iommu_ops;  in iommu_sva_get_pasid()  local
3190     if (!ops || !ops->sva_get_pasid) {  in iommu_sva_get_pasid()
3194     return ops->sva_get_pasid(handle);  in iommu_sva_get_pasid()