58 static void msi_domain_free_locked(struct device *dev, struct msi_ctrl *ctrl);
59 static unsigned int msi_domain_get_hwsize(struct device *dev, unsigned int domid);
60 static inline int msi_sysfs_create_group(struct device *dev);
74 static struct msi_desc *msi_alloc_desc(struct device *dev, int nvec,  in msi_alloc_desc()  argument
82 	desc->dev = dev;  in msi_alloc_desc()
100 static int msi_insert_desc(struct device *dev, struct msi_desc *desc,  in msi_insert_desc()  argument
103 	struct msi_device_data *md = dev->msi.data;  in msi_insert_desc()
108 	hwsize = msi_domain_get_hwsize(dev, domid);  in msi_insert_desc()
148 int msi_domain_insert_msi_desc(struct device *dev, unsigned int domid,  in msi_domain_insert_msi_desc()  argument
153 	lockdep_assert_held(&dev->msi.data->mutex);  in msi_domain_insert_msi_desc()
155 	desc = msi_alloc_desc(dev, init_desc->nvec_used, init_desc->affinity);  in msi_domain_insert_msi_desc()
162 	return msi_insert_desc(dev, desc, domid, init_desc->msi_index);  in msi_domain_insert_msi_desc()
179 static bool msi_ctrl_valid(struct device *dev, struct msi_ctrl *ctrl)  in msi_ctrl_valid()  argument
184 			 (dev->msi.domain &&  in msi_ctrl_valid()
185 			  !dev->msi.data->__domains[ctrl->domid].domain)))  in msi_ctrl_valid()
188 	hwsize = msi_domain_get_hwsize(dev, ctrl->domid);  in msi_ctrl_valid()
196 static void msi_domain_free_descs(struct device *dev, struct msi_ctrl *ctrl)  in msi_domain_free_descs()  argument
202 	lockdep_assert_held(&dev->msi.data->mutex);  in msi_domain_free_descs()
204 	if (!msi_ctrl_valid(dev, ctrl))  in msi_domain_free_descs()
207 	xa = &dev->msi.data->__domains[ctrl->domid].store;  in msi_domain_free_descs()
225 void msi_domain_free_msi_descs_range(struct device *dev, unsigned int domid,  in msi_domain_free_msi_descs_range()  argument
234 	msi_domain_free_descs(dev, &ctrl);  in msi_domain_free_msi_descs_range()
244 static int msi_domain_add_simple_msi_descs(struct device *dev, struct msi_ctrl *ctrl)  in msi_domain_add_simple_msi_descs()  argument
250 	lockdep_assert_held(&dev->msi.data->mutex);  in msi_domain_add_simple_msi_descs()
252 	if (!msi_ctrl_valid(dev, ctrl))  in msi_domain_add_simple_msi_descs()
256 		desc = msi_alloc_desc(dev, 1, NULL);  in msi_domain_add_simple_msi_descs()
259 		ret = msi_insert_desc(dev, desc, ctrl->domid, idx);  in msi_domain_add_simple_msi_descs()
268 	msi_domain_free_descs(dev, ctrl);  in msi_domain_add_simple_msi_descs()
285 static void msi_device_data_release(struct device *dev, void *res)  in msi_device_data_release()  argument
291 		msi_remove_device_irq_domain(dev, i);  in msi_device_data_release()
295 	dev->msi.data = NULL;  in msi_device_data_release()
308 int msi_setup_device_data(struct device *dev)  in msi_setup_device_data()  argument
313 	if (dev->msi.data)  in msi_setup_device_data()
320 	ret = msi_sysfs_create_group(dev);  in msi_setup_device_data()
335 	if (dev->msi.domain && !irq_domain_is_msi_parent(dev->msi.domain))  in msi_setup_device_data()
336 		md->__domains[MSI_DEFAULT_DOMAIN].domain = dev->msi.domain;  in msi_setup_device_data()
339 	dev->msi.data = md;  in msi_setup_device_data()
340 	devres_add(dev, md);  in msi_setup_device_data()
348 void msi_lock_descs(struct device *dev)  in msi_lock_descs()  argument
350 	mutex_lock(&dev->msi.data->mutex);  in msi_lock_descs()
358 void msi_unlock_descs(struct device *dev)  in msi_unlock_descs()  argument
361 	dev->msi.data->__iter_idx = MSI_XA_MAX_INDEX;  in msi_unlock_descs()
362 	mutex_unlock(&dev->msi.data->mutex);  in msi_unlock_descs()
392 struct msi_desc *msi_domain_first_desc(struct device *dev, unsigned int domid,  in msi_domain_first_desc()  argument
395 	struct msi_device_data *md = dev->msi.data;  in msi_domain_first_desc()
421 struct msi_desc *msi_next_desc(struct device *dev, unsigned int domid,  in msi_next_desc()  argument
424 	struct msi_device_data *md = dev->msi.data;  in msi_next_desc()
447 unsigned int msi_domain_get_virq(struct device *dev, unsigned int domid, unsigned int index)  in msi_domain_get_virq()  argument
454 	if (!dev->msi.data)  in msi_domain_get_virq()
461 	if (dev_is_pci(dev) && domid == MSI_DEFAULT_DOMAIN)  in msi_domain_get_virq()
462 		pcimsi = to_pci_dev(dev)->msi_enabled;  in msi_domain_get_virq()
464 	msi_lock_descs(dev);  in msi_domain_get_virq()
465 	xa = &dev->msi.data->__domains[domid].store;  in msi_domain_get_virq()
481 	msi_unlock_descs(dev);  in msi_domain_get_virq()
496 static inline int msi_sysfs_create_group(struct device *dev)  in msi_sysfs_create_group()  argument
498 	return devm_device_add_group(dev, &msi_irqs_group);  in msi_sysfs_create_group()
501 static ssize_t msi_mode_show(struct device *dev, struct device_attribute *attr,  in msi_mode_show()  argument
505 	bool is_msix = dev_is_pci(dev) ? to_pci_dev(dev)->msix_enabled : false;  in msi_mode_show()
510 static void msi_sysfs_remove_desc(struct device *dev, struct msi_desc *desc)  in msi_sysfs_remove_desc()  argument
521 			sysfs_remove_file_from_group(&dev->kobj, &attrs[i].attr, msi_irqs_group.name);  in msi_sysfs_remove_desc()
527 static int msi_sysfs_populate_desc(struct device *dev, struct msi_desc *desc)  in msi_sysfs_populate_desc()  argument
548 		ret = sysfs_add_file_to_group(&dev->kobj, &attrs[i].attr, msi_irqs_group.name);  in msi_sysfs_populate_desc()
557 	msi_sysfs_remove_desc(dev, desc);  in msi_sysfs_populate_desc()
566 int msi_device_populate_sysfs(struct device *dev)  in msi_device_populate_sysfs()  argument
571 	msi_for_each_desc(desc, dev, MSI_DESC_ASSOCIATED) {  in msi_device_populate_sysfs()
574 		ret = msi_sysfs_populate_desc(dev, desc);  in msi_device_populate_sysfs()
586 void msi_device_destroy_sysfs(struct device *dev)  in msi_device_destroy_sysfs()  argument
590 	msi_for_each_desc(desc, dev, MSI_DESC_ALL)  in msi_device_destroy_sysfs()
591 		msi_sysfs_remove_desc(dev, desc);  in msi_device_destroy_sysfs()
595 static inline int msi_sysfs_create_group(struct device *dev) { return 0; }  in msi_sysfs_create_group()  argument
596 static inline int msi_sysfs_populate_desc(struct device *dev, struct msi_desc *desc) { return 0; }  in msi_sysfs_populate_desc()  argument
597 static inline void msi_sysfs_remove_desc(struct device *dev, struct msi_desc *desc) { }  in msi_sysfs_remove_desc()  argument
600 static struct irq_domain *msi_get_device_domain(struct device *dev, unsigned int domid)  in msi_get_device_domain()  argument
604 	lockdep_assert_held(&dev->msi.data->mutex);  in msi_get_device_domain()
609 	domain = dev->msi.data->__domains[domid].domain;  in msi_get_device_domain()
619 static unsigned int msi_domain_get_hwsize(struct device *dev, unsigned int domid)  in msi_domain_get_hwsize()  argument
624 	domain = msi_get_device_domain(dev, domid);  in msi_domain_get_hwsize()
773 static int msi_domain_ops_prepare(struct irq_domain *domain, struct device *dev,  in msi_domain_ops_prepare()  argument
915 bool msi_parent_init_dev_msi_info(struct device *dev, struct irq_domain *domain,  in msi_parent_init_dev_msi_info()  argument
925 	return parent->msi_parent_ops->init_dev_msi_info(dev, parent, msi_parent_domain,  in msi_parent_init_dev_msi_info()
976 bool msi_create_device_irq_domain(struct device *dev, unsigned int domid,  in msi_create_device_irq_domain()  argument
981 	struct irq_domain *domain, *parent = dev->msi.domain;  in msi_create_device_irq_domain()
1004 		 pops->prefix ? : "", bundle->chip.name, dev_name(dev));  in msi_create_device_irq_domain()
1016 		fwnode = dev->fwnode;  in msi_create_device_irq_domain()
1023 	if (msi_setup_device_data(dev))  in msi_create_device_irq_domain()
1026 	msi_lock_descs(dev);  in msi_create_device_irq_domain()
1028 	if (WARN_ON_ONCE(msi_get_device_domain(dev, domid)))  in msi_create_device_irq_domain()
1031 	if (!pops->init_dev_msi_info(dev, parent, parent, &bundle->info))  in msi_create_device_irq_domain()
1038 	domain->dev = dev;  in msi_create_device_irq_domain()
1039 	dev->msi.data->__domains[domid].domain = domain;  in msi_create_device_irq_domain()
1040 	msi_unlock_descs(dev);  in msi_create_device_irq_domain()
1044 	msi_unlock_descs(dev);  in msi_create_device_irq_domain()
1057 void msi_remove_device_irq_domain(struct device *dev, unsigned int domid)  in msi_remove_device_irq_domain()  argument
1063 	msi_lock_descs(dev);  in msi_remove_device_irq_domain()
1065 	domain = msi_get_device_domain(dev, domid);  in msi_remove_device_irq_domain()
1070 	dev->msi.data->__domains[domid].domain = NULL;  in msi_remove_device_irq_domain()
1079 	msi_unlock_descs(dev);  in msi_remove_device_irq_domain()
1090 bool msi_match_device_irq_domain(struct device *dev, unsigned int domid,  in msi_match_device_irq_domain()  argument
1097 	msi_lock_descs(dev);  in msi_match_device_irq_domain()
1098 	domain = msi_get_device_domain(dev, domid);  in msi_match_device_irq_domain()
1103 	msi_unlock_descs(dev);  in msi_match_device_irq_domain()
1107 static int msi_domain_prepare_irqs(struct irq_domain *domain, struct device *dev,  in msi_domain_prepare_irqs()  argument
1113 	return ops->msi_prepare(domain, dev, nvec, arg);  in msi_domain_prepare_irqs()
1129 				       struct device *dev)  in msi_check_reservation_mode()  argument
1153 	desc = msi_first_desc(dev, MSI_DESC_ALL);  in msi_check_reservation_mode()
1222 static int __msi_domain_alloc_irqs(struct device *dev, struct irq_domain *domain,  in __msi_domain_alloc_irqs()  argument
1225 	struct xarray *xa = &dev->msi.data->__domains[ctrl->domid].store;  in __msi_domain_alloc_irqs()
1234 	ret = msi_domain_prepare_irqs(domain, dev, ctrl->nirqs, &arg);  in __msi_domain_alloc_irqs()
1250 	if (msi_check_reservation_mode(domain, info, dev))  in __msi_domain_alloc_irqs()
1267 					       dev_to_node(dev), &arg, false,  in __msi_domain_alloc_irqs()
1274 			irq_debugfs_copy_devname(virq + i, dev);  in __msi_domain_alloc_irqs()
1280 			ret = msi_sysfs_populate_desc(dev, desc);  in __msi_domain_alloc_irqs()
1289 static int msi_domain_alloc_simple_msi_descs(struct device *dev,  in msi_domain_alloc_simple_msi_descs()  argument
1296 	return msi_domain_add_simple_msi_descs(dev, ctrl);  in msi_domain_alloc_simple_msi_descs()
1299 static int __msi_domain_alloc_locked(struct device *dev, struct msi_ctrl *ctrl)  in __msi_domain_alloc_locked()  argument
1306 	if (!msi_ctrl_valid(dev, ctrl))  in __msi_domain_alloc_locked()
1309 	domain = msi_get_device_domain(dev, ctrl->domid);  in __msi_domain_alloc_locked()
1315 	ret = msi_domain_alloc_simple_msi_descs(dev, info, ctrl);  in __msi_domain_alloc_locked()
1321 		return ops->domain_alloc_irqs(domain, dev, ctrl->nirqs);  in __msi_domain_alloc_locked()
1323 	return __msi_domain_alloc_irqs(dev, domain, ctrl);  in __msi_domain_alloc_locked()
1326 static int msi_domain_alloc_locked(struct device *dev, struct msi_ctrl *ctrl)  in msi_domain_alloc_locked()  argument
1328 	int ret = __msi_domain_alloc_locked(dev, ctrl);  in msi_domain_alloc_locked()
1331 		msi_domain_free_locked(dev, ctrl);  in msi_domain_alloc_locked()
1349 int msi_domain_alloc_irqs_range_locked(struct device *dev, unsigned int domid,  in msi_domain_alloc_irqs_range_locked()  argument
1359 	return msi_domain_alloc_locked(dev, &ctrl);  in msi_domain_alloc_irqs_range_locked()
1372 int msi_domain_alloc_irqs_range(struct device *dev, unsigned int domid,  in msi_domain_alloc_irqs_range()  argument
1377 	msi_lock_descs(dev);  in msi_domain_alloc_irqs_range()
1378 	ret = msi_domain_alloc_irqs_range_locked(dev, domid, first, last);  in msi_domain_alloc_irqs_range()
1379 	msi_unlock_descs(dev);  in msi_domain_alloc_irqs_range()
1398 int msi_domain_alloc_irqs_all_locked(struct device *dev, unsigned int domid, int nirqs)  in msi_domain_alloc_irqs_all_locked()  argument
1403 		.last	= msi_domain_get_hwsize(dev, domid) - 1,  in msi_domain_alloc_irqs_all_locked()
1407 	return msi_domain_alloc_locked(dev, &ctrl);  in msi_domain_alloc_irqs_all_locked()
1410 static struct msi_map __msi_domain_alloc_irq_at(struct device *dev, unsigned int domid,  in __msi_domain_alloc_irq_at()  argument
1421 	domain = msi_get_device_domain(dev, domid);  in __msi_domain_alloc_irq_at()
1427 	desc = msi_alloc_desc(dev, 1, affdesc);  in __msi_domain_alloc_irq_at()
1436 	ret = msi_insert_desc(dev, desc, domid, index);  in __msi_domain_alloc_irq_at()
1444 	ret = __msi_domain_alloc_irqs(dev, domain, &ctrl);  in __msi_domain_alloc_irq_at()
1447 		msi_domain_free_locked(dev, &ctrl);  in __msi_domain_alloc_irq_at()
1480 struct msi_map msi_domain_alloc_irq_at(struct device *dev, unsigned int domid, unsigned int index,  in msi_domain_alloc_irq_at()  argument
1486 	msi_lock_descs(dev);  in msi_domain_alloc_irq_at()
1487 	map = __msi_domain_alloc_irq_at(dev, domid, index, affdesc, icookie);  in msi_domain_alloc_irq_at()
1488 	msi_unlock_descs(dev);  in msi_domain_alloc_irq_at()
1518 	struct device *dev = domain->dev;  in msi_device_domain_alloc_wired()  local
1521 	if (WARN_ON_ONCE(!dev || domain->bus_token != DOMAIN_BUS_WIRED_TO_MSI))  in msi_device_domain_alloc_wired()
1526 	msi_lock_descs(dev);  in msi_device_domain_alloc_wired()
1527 	if (WARN_ON_ONCE(msi_get_device_domain(dev, domid) != domain))  in msi_device_domain_alloc_wired()
1530 		map = __msi_domain_alloc_irq_at(dev, domid, MSI_ANY_INDEX, NULL, &icookie);  in msi_device_domain_alloc_wired()
1531 	msi_unlock_descs(dev);  in msi_device_domain_alloc_wired()
1536 static void __msi_domain_free_irqs(struct device *dev, struct irq_domain *domain,  in __msi_domain_free_irqs()  argument
1539 	struct xarray *xa = &dev->msi.data->__domains[ctrl->domid].store;  in __msi_domain_free_irqs()
1560 			msi_sysfs_remove_desc(dev, desc);  in __msi_domain_free_irqs()
1565 static void msi_domain_free_locked(struct device *dev, struct msi_ctrl *ctrl)  in msi_domain_free_locked()  argument
1571 	if (!msi_ctrl_valid(dev, ctrl))  in msi_domain_free_locked()
1574 	domain = msi_get_device_domain(dev, ctrl->domid);  in msi_domain_free_locked()
1582 		ops->domain_free_irqs(domain, dev);  in msi_domain_free_locked()
1584 		__msi_domain_free_irqs(dev, domain, ctrl);  in msi_domain_free_locked()
1587 		ops->msi_post_free(domain, dev);  in msi_domain_free_locked()
1590 		msi_domain_free_descs(dev, ctrl);  in msi_domain_free_locked()
1602 void msi_domain_free_irqs_range_locked(struct device *dev, unsigned int domid,  in msi_domain_free_irqs_range_locked()  argument
1610 	msi_domain_free_locked(dev, &ctrl);  in msi_domain_free_irqs_range_locked()
1622 void msi_domain_free_irqs_range(struct device *dev, unsigned int domid,  in msi_domain_free_irqs_range()  argument
1625 	msi_lock_descs(dev);  in msi_domain_free_irqs_range()
1626 	msi_domain_free_irqs_range_locked(dev, domid, first, last);  in msi_domain_free_irqs_range()
1627 	msi_unlock_descs(dev);  in msi_domain_free_irqs_range()
1642 void msi_domain_free_irqs_all_locked(struct device *dev, unsigned int domid)  in msi_domain_free_irqs_all_locked()  argument
1644 	msi_domain_free_irqs_range_locked(dev, domid, 0,  in msi_domain_free_irqs_all_locked()
1645 					  msi_domain_get_hwsize(dev, domid) - 1);  in msi_domain_free_irqs_all_locked()
1655 void msi_domain_free_irqs_all(struct device *dev, unsigned int domid)  in msi_domain_free_irqs_all()  argument
1657 	msi_lock_descs(dev);  in msi_domain_free_irqs_all()
1658 	msi_domain_free_irqs_all_locked(dev, domid);  in msi_domain_free_irqs_all()
1659 	msi_unlock_descs(dev);  in msi_domain_free_irqs_all()
1673 	struct device *dev = domain->dev;  in msi_device_domain_free_wired()  local
1675 	if (WARN_ON_ONCE(!dev || !desc || domain->bus_token != DOMAIN_BUS_WIRED_TO_MSI))  in msi_device_domain_free_wired()
1678 	msi_lock_descs(dev);  in msi_device_domain_free_wired()
1679 	if (!WARN_ON_ONCE(msi_get_device_domain(dev, MSI_DEFAULT_DOMAIN) != domain)) {  in msi_device_domain_free_wired()
1680 		msi_domain_free_irqs_range_locked(dev, MSI_DEFAULT_DOMAIN, desc->msi_index,  in msi_device_domain_free_wired()
1683 	msi_unlock_descs(dev);  in msi_device_domain_free_wired()
1713 bool msi_device_has_isolated_msi(struct device *dev)  in msi_device_has_isolated_msi()  argument
1715 	struct irq_domain *domain = dev_get_msi_domain(dev);  in msi_device_has_isolated_msi()