28 static int irq_domain_alloc_irqs_locked(struct irq_domain *domain, int irq_base,
31 static void irq_domain_check_hierarchy(struct irq_domain *domain);
137 	struct irq_domain *domain;  in __irq_domain_create()  local
146 	domain = kzalloc_node(struct_size(domain, revmap, size),  in __irq_domain_create()
148 	if (!domain)  in __irq_domain_create()
157 			domain->fwnode = fwnode;  in __irq_domain_create()
158 			domain->name = kstrdup(fwid->name, GFP_KERNEL);  in __irq_domain_create()
159 			if (!domain->name) {  in __irq_domain_create()
160 				kfree(domain);  in __irq_domain_create()
163 			domain->flags |= IRQ_DOMAIN_NAME_ALLOCATED;  in __irq_domain_create()
166 			domain->fwnode = fwnode;  in __irq_domain_create()
167 			domain->name = fwid->name;  in __irq_domain_create()
181 			kfree(domain);  in __irq_domain_create()
187 		domain->name = name;  in __irq_domain_create()
188 		domain->fwnode = fwnode;  in __irq_domain_create()
189 		domain->flags |= IRQ_DOMAIN_NAME_ALLOCATED;  in __irq_domain_create()
192 	if (!domain->name) {  in __irq_domain_create()
195 		domain->name = kasprintf(GFP_KERNEL, "unknown-%d",  in __irq_domain_create()
197 		if (!domain->name) {  in __irq_domain_create()
198 			kfree(domain);  in __irq_domain_create()
201 		domain->flags |= IRQ_DOMAIN_NAME_ALLOCATED;  in __irq_domain_create()
208 	INIT_RADIX_TREE(&domain->revmap_tree, GFP_KERNEL);  in __irq_domain_create()
209 	mutex_init(&domain->revmap_mutex);  in __irq_domain_create()
210 	domain->ops = ops;  in __irq_domain_create()
211 	domain->host_data = host_data;  in __irq_domain_create()
212 	domain->hwirq_max = hwirq_max;  in __irq_domain_create()
215 		domain->flags |= IRQ_DOMAIN_FLAG_NO_MAP;  in __irq_domain_create()
218 	domain->revmap_size = size;  in __irq_domain_create()
220 	irq_domain_check_hierarchy(domain);  in __irq_domain_create()
222 	return domain;  in __irq_domain_create()
225 static void __irq_domain_publish(struct irq_domain *domain)  in __irq_domain_publish()  argument
228 	debugfs_add_domain_dir(domain);  in __irq_domain_publish()
229 	list_add(&domain->link, &irq_domain_list);  in __irq_domain_publish()
232 	pr_debug("Added domain %s\n", domain->name);  in __irq_domain_publish()
253 	struct irq_domain *domain;  in __irq_domain_add()  local
255 	domain = __irq_domain_create(fwnode, size, hwirq_max, direct_max,  in __irq_domain_add()
257 	if (domain)  in __irq_domain_add()
258 		__irq_domain_publish(domain);  in __irq_domain_add()
260 	return domain;  in __irq_domain_add()
272 void irq_domain_remove(struct irq_domain *domain)  in irq_domain_remove()  argument
275 	debugfs_remove_domain_dir(domain);  in irq_domain_remove()
277 	WARN_ON(!radix_tree_empty(&domain->revmap_tree));  in irq_domain_remove()
279 	list_del(&domain->link);  in irq_domain_remove()
284 	if (unlikely(irq_default_domain == domain))  in irq_domain_remove()
289 	pr_debug("Removed domain %s\n", domain->name);  in irq_domain_remove()
291 	fwnode_dev_initialized(domain->fwnode, false);  in irq_domain_remove()
292 	fwnode_handle_put(domain->fwnode);  in irq_domain_remove()
293 	if (domain->flags & IRQ_DOMAIN_NAME_ALLOCATED)  in irq_domain_remove()
294 		kfree(domain->name);  in irq_domain_remove()
295 	kfree(domain);  in irq_domain_remove()
299 void irq_domain_update_bus_token(struct irq_domain *domain,  in irq_domain_update_bus_token()  argument
304 	if (domain->bus_token == bus_token)  in irq_domain_update_bus_token()
309 	domain->bus_token = bus_token;  in irq_domain_update_bus_token()
311 	name = kasprintf(GFP_KERNEL, "%s-%d", domain->name, bus_token);  in irq_domain_update_bus_token()
317 	debugfs_remove_domain_dir(domain);  in irq_domain_update_bus_token()
319 	if (domain->flags & IRQ_DOMAIN_NAME_ALLOCATED)  in irq_domain_update_bus_token()
320 		kfree(domain->name);  in irq_domain_update_bus_token()
322 		domain->flags |= IRQ_DOMAIN_NAME_ALLOCATED;  in irq_domain_update_bus_token()
324 	domain->name = name;  in irq_domain_update_bus_token()
325 	debugfs_add_domain_dir(domain);  in irq_domain_update_bus_token()
355 	struct irq_domain *domain;  in irq_domain_create_simple()  local
357 	domain = __irq_domain_add(fwnode, size, size, 0, ops, host_data);  in irq_domain_create_simple()
358 	if (!domain)  in irq_domain_create_simple()
370 		irq_domain_associate_many(domain, first_irq, 0, size);  in irq_domain_create_simple()
373 	return domain;  in irq_domain_create_simple()
411 	struct irq_domain *domain;  in irq_domain_create_legacy()  local
413 	domain = __irq_domain_add(fwnode, first_hwirq + size, first_hwirq + size, 0, ops, host_data);  in irq_domain_create_legacy()
414 	if (domain)  in irq_domain_create_legacy()
415 		irq_domain_associate_many(domain, first_irq, first_hwirq, size);  in irq_domain_create_legacy()
417 	return domain;  in irq_domain_create_legacy()
497 void irq_set_default_host(struct irq_domain *domain)  in irq_set_default_host()  argument
499 	pr_debug("Default domain set to @0x%p\n", domain);  in irq_set_default_host()
501 	irq_default_domain = domain;  in irq_set_default_host()
520 static bool irq_domain_is_nomap(struct irq_domain *domain)  in irq_domain_is_nomap()  argument
523 	       (domain->flags & IRQ_DOMAIN_FLAG_NO_MAP);  in irq_domain_is_nomap()
526 static void irq_domain_clear_mapping(struct irq_domain *domain,  in irq_domain_clear_mapping()  argument
529 	if (irq_domain_is_nomap(domain))  in irq_domain_clear_mapping()
532 	mutex_lock(&domain->revmap_mutex);  in irq_domain_clear_mapping()
533 	if (hwirq < domain->revmap_size)  in irq_domain_clear_mapping()
534 		rcu_assign_pointer(domain->revmap[hwirq], NULL);  in irq_domain_clear_mapping()
536 		radix_tree_delete(&domain->revmap_tree, hwirq);  in irq_domain_clear_mapping()
537 	mutex_unlock(&domain->revmap_mutex);  in irq_domain_clear_mapping()
540 static void irq_domain_set_mapping(struct irq_domain *domain,  in irq_domain_set_mapping()  argument
544 	if (irq_domain_is_nomap(domain))  in irq_domain_set_mapping()
547 	mutex_lock(&domain->revmap_mutex);  in irq_domain_set_mapping()
548 	if (hwirq < domain->revmap_size)  in irq_domain_set_mapping()
549 		rcu_assign_pointer(domain->revmap[hwirq], irq_data);  in irq_domain_set_mapping()
551 		radix_tree_insert(&domain->revmap_tree, hwirq, irq_data);  in irq_domain_set_mapping()
552 	mutex_unlock(&domain->revmap_mutex);  in irq_domain_set_mapping()
555 static void irq_domain_disassociate(struct irq_domain *domain, unsigned int irq)  in irq_domain_disassociate()  argument
560 	if (WARN(!irq_data || irq_data->domain != domain,  in irq_domain_disassociate()
577 	if (domain->ops->unmap)  in irq_domain_disassociate()
578 		domain->ops->unmap(domain, irq);  in irq_domain_disassociate()
581 	irq_data->domain = NULL;  in irq_domain_disassociate()
583 	domain->mapcount--;  in irq_domain_disassociate()
586 	irq_domain_clear_mapping(domain, hwirq);  in irq_domain_disassociate()
591 static int irq_domain_associate_locked(struct irq_domain *domain, unsigned int virq,  in irq_domain_associate_locked()  argument
597 	if (WARN(hwirq >= domain->hwirq_max,  in irq_domain_associate_locked()
598 		 "error: hwirq 0x%x is too large for %s\n", (int)hwirq, domain->name))  in irq_domain_associate_locked()
602 	if (WARN(irq_data->domain, "error: virq%i is already associated", virq))  in irq_domain_associate_locked()
606 	irq_data->domain = domain;  in irq_domain_associate_locked()
607 	if (domain->ops->map) {  in irq_domain_associate_locked()
608 		ret = domain->ops->map(domain, virq, hwirq);  in irq_domain_associate_locked()
617 				       domain->name, hwirq, virq, ret);  in irq_domain_associate_locked()
619 			irq_data->domain = NULL;  in irq_domain_associate_locked()
625 		if (!domain->name && irq_data->chip)  in irq_domain_associate_locked()
626 			domain->name = irq_data->chip->name;  in irq_domain_associate_locked()
629 	domain->mapcount++;  in irq_domain_associate_locked()
630 	irq_domain_set_mapping(domain, hwirq, irq_data);  in irq_domain_associate_locked()
637 int irq_domain_associate(struct irq_domain *domain, unsigned int virq,  in irq_domain_associate()  argument
643 	ret = irq_domain_associate_locked(domain, virq, hwirq);  in irq_domain_associate()
650 void irq_domain_associate_many(struct irq_domain *domain, unsigned int irq_base,  in irq_domain_associate_many()  argument
656 	of_node = irq_domain_get_of_node(domain);  in irq_domain_associate_many()
661 		irq_domain_associate(domain, irq_base + i, hwirq_base + i);  in irq_domain_associate_many()
677 unsigned int irq_create_direct_mapping(struct irq_domain *domain)  in irq_create_direct_mapping()  argument
682 	if (domain == NULL)  in irq_create_direct_mapping()
683 		domain = irq_default_domain;  in irq_create_direct_mapping()
685 	of_node = irq_domain_get_of_node(domain);  in irq_create_direct_mapping()
691 	if (virq >= domain->hwirq_max) {  in irq_create_direct_mapping()
693 			domain->hwirq_max);  in irq_create_direct_mapping()
699 	if (irq_domain_associate(domain, virq, virq)) {  in irq_create_direct_mapping()
709 static unsigned int irq_create_mapping_affinity_locked(struct irq_domain *domain,  in irq_create_mapping_affinity_locked()  argument
713 	struct device_node *of_node = irq_domain_get_of_node(domain);  in irq_create_mapping_affinity_locked()
716 	pr_debug("irq_create_mapping(0x%p, 0x%lx)\n", domain, hwirq);  in irq_create_mapping_affinity_locked()
726 	if (irq_domain_associate_locked(domain, virq, hwirq)) {  in irq_create_mapping_affinity_locked()
748 unsigned int irq_create_mapping_affinity(struct irq_domain *domain,  in irq_create_mapping_affinity()  argument
755 	if (domain == NULL)  in irq_create_mapping_affinity()
756 		domain = irq_default_domain;  in irq_create_mapping_affinity()
757 	if (domain == NULL) {  in irq_create_mapping_affinity()
765 	virq = irq_find_mapping(domain, hwirq);  in irq_create_mapping_affinity()
771 	virq = irq_create_mapping_affinity_locked(domain, hwirq, affinity);  in irq_create_mapping_affinity()
812 	struct irq_domain *domain;  in irq_create_fwspec_mapping()  local
819 		domain = irq_find_matching_fwspec(fwspec, DOMAIN_BUS_WIRED);  in irq_create_fwspec_mapping()
820 		if (!domain)  in irq_create_fwspec_mapping()
821 			domain = irq_find_matching_fwspec(fwspec, DOMAIN_BUS_ANY);  in irq_create_fwspec_mapping()
823 		domain = irq_default_domain;  in irq_create_fwspec_mapping()
826 	if (!domain) {  in irq_create_fwspec_mapping()
832 	if (irq_domain_translate(domain, fwspec, &hwirq, &type))  in irq_create_fwspec_mapping()
848 	virq = irq_find_mapping(domain, hwirq);  in irq_create_fwspec_mapping()
879 	if (irq_domain_is_hierarchy(domain)) {  in irq_create_fwspec_mapping()
880 		virq = irq_domain_alloc_irqs_locked(domain, -1, 1, NUMA_NO_NODE,  in irq_create_fwspec_mapping()
888 		virq = irq_create_mapping_affinity_locked(domain, hwirq, NULL);  in irq_create_fwspec_mapping()
926 	struct irq_domain *domain;  in irq_dispose_mapping()  local
931 	domain = irq_data->domain;  in irq_dispose_mapping()
932 	if (WARN_ON(domain == NULL))  in irq_dispose_mapping()
935 	if (irq_domain_is_hierarchy(domain)) {  in irq_dispose_mapping()
938 		irq_domain_disassociate(domain, virq);  in irq_dispose_mapping()
952 struct irq_desc *__irq_resolve_mapping(struct irq_domain *domain,  in __irq_resolve_mapping()  argument
960 	if (domain == NULL)  in __irq_resolve_mapping()
961 		domain = irq_default_domain;  in __irq_resolve_mapping()
962 	if (domain == NULL)  in __irq_resolve_mapping()
965 	if (irq_domain_is_nomap(domain)) {  in __irq_resolve_mapping()
966 		if (hwirq < domain->hwirq_max) {  in __irq_resolve_mapping()
967 			data = irq_domain_get_irq_data(domain, hwirq);  in __irq_resolve_mapping()
979 	if (hwirq < domain->revmap_size)  in __irq_resolve_mapping()
980 		data = rcu_dereference(domain->revmap[hwirq]);  in __irq_resolve_mapping()
982 		data = radix_tree_lookup(&domain->revmap_tree, hwirq);  in __irq_resolve_mapping()
1159 	struct irq_domain *domain;  in irq_domain_create_hierarchy()  local
1162 		domain = __irq_domain_create(fwnode, size, size, 0, ops, host_data);  in irq_domain_create_hierarchy()
1164 		domain = __irq_domain_create(fwnode, 0, ~0, 0, ops, host_data);  in irq_domain_create_hierarchy()
1166 	if (domain) {  in irq_domain_create_hierarchy()
1167 		domain->parent = parent;  in irq_domain_create_hierarchy()
1168 		domain->flags |= flags;  in irq_domain_create_hierarchy()
1170 		__irq_domain_publish(domain);  in irq_domain_create_hierarchy()
1173 	return domain;  in irq_domain_create_hierarchy()
1182 		struct irq_domain *domain = data->domain;  in irq_domain_insert_irq()  local
1184 		domain->mapcount++;  in irq_domain_insert_irq()
1185 		irq_domain_set_mapping(domain, data->hwirq, data);  in irq_domain_insert_irq()
1188 		if (!domain->name && data->chip)  in irq_domain_insert_irq()
1189 			domain->name = data->chip->name;  in irq_domain_insert_irq()
1205 		struct irq_domain *domain = data->domain;  in irq_domain_remove_irq()  local
1208 		domain->mapcount--;  in irq_domain_remove_irq()
1209 		irq_domain_clear_mapping(domain, hwirq);  in irq_domain_remove_irq()
1213 static struct irq_data *irq_domain_insert_irq_data(struct irq_domain *domain,  in irq_domain_insert_irq_data()  argument
1224 		irq_data->domain = domain;  in irq_domain_insert_irq_data()
1250 		irq_data->domain = NULL;  in irq_domain_free_irq_data()
1269 int irq_domain_disconnect_hierarchy(struct irq_domain *domain,  in irq_domain_disconnect_hierarchy()  argument
1274 	irqd = irq_domain_get_irq_data(domain, virq);  in irq_domain_disconnect_hierarchy()
1321 		virq, tail->parent_data->domain->name);  in irq_domain_trim_hierarchy()
1332 static int irq_domain_alloc_irq_data(struct irq_domain *domain,  in irq_domain_alloc_irq_data()  argument
1342 		irq_data->domain = domain;  in irq_domain_alloc_irq_data()
1344 		for (parent = domain->parent; parent; parent = parent->parent) {  in irq_domain_alloc_irq_data()
1361 struct irq_data *irq_domain_get_irq_data(struct irq_domain *domain,  in irq_domain_get_irq_data()  argument
1368 		if (irq_data->domain == domain)  in irq_domain_get_irq_data()
1383 int irq_domain_set_hwirq_and_chip(struct irq_domain *domain, unsigned int virq,  in irq_domain_set_hwirq_and_chip()  argument
1388 	struct irq_data *irq_data = irq_domain_get_irq_data(domain, virq);  in irq_domain_set_hwirq_and_chip()
1412 void irq_domain_set_info(struct irq_domain *domain, unsigned int virq,  in irq_domain_set_info()  argument
1417 	irq_domain_set_hwirq_and_chip(domain, virq, hwirq, chip, chip_data);  in irq_domain_set_info()
1429 void irq_domain_free_irqs_common(struct irq_domain *domain, unsigned int virq,  in irq_domain_free_irqs_common()  argument
1436 		irq_data = irq_domain_get_irq_data(domain, virq + i);  in irq_domain_free_irqs_common()
1440 	irq_domain_free_irqs_parent(domain, virq, nr_irqs);  in irq_domain_free_irqs_common()
1450 void irq_domain_free_irqs_top(struct irq_domain *domain, unsigned int virq,  in irq_domain_free_irqs_top()  argument
1459 	irq_domain_free_irqs_common(domain, virq, nr_irqs);  in irq_domain_free_irqs_top()
1462 static void irq_domain_free_irqs_hierarchy(struct irq_domain *domain,  in irq_domain_free_irqs_hierarchy()  argument
1468 	if (!domain->ops->free)  in irq_domain_free_irqs_hierarchy()
1472 		if (irq_domain_get_irq_data(domain, irq_base + i))  in irq_domain_free_irqs_hierarchy()
1473 			domain->ops->free(domain, irq_base + i, 1);  in irq_domain_free_irqs_hierarchy()
1477 int irq_domain_alloc_irqs_hierarchy(struct irq_domain *domain,  in irq_domain_alloc_irqs_hierarchy()  argument
1481 	if (!domain->ops->alloc) {  in irq_domain_alloc_irqs_hierarchy()
1486 	return domain->ops->alloc(domain, irq_base, nr_irqs, arg);  in irq_domain_alloc_irqs_hierarchy()
1489 static int irq_domain_alloc_irqs_locked(struct irq_domain *domain, int irq_base,  in irq_domain_alloc_irqs_locked()  argument
1507 	if (irq_domain_alloc_irq_data(domain, virq, nr_irqs)) {  in irq_domain_alloc_irqs_locked()
1513 	ret = irq_domain_alloc_irqs_hierarchy(domain, virq, nr_irqs, arg);  in irq_domain_alloc_irqs_locked()
1557 int __irq_domain_alloc_irqs(struct irq_domain *domain, int irq_base,  in __irq_domain_alloc_irqs()  argument
1563 	if (domain == NULL) {  in __irq_domain_alloc_irqs()
1564 		domain = irq_default_domain;  in __irq_domain_alloc_irqs()
1565 		if (WARN(!domain, "domain is NULL; cannot allocate IRQ\n"))  in __irq_domain_alloc_irqs()
1570 	ret = irq_domain_alloc_irqs_locked(domain, irq_base, nr_irqs, node, arg,  in __irq_domain_alloc_irqs()
1583 	if (irq_domain_is_nomap(d->domain))  in irq_domain_fix_revmap()
1587 	mutex_lock(&d->domain->revmap_mutex);  in irq_domain_fix_revmap()
1588 	if (d->hwirq < d->domain->revmap_size) {  in irq_domain_fix_revmap()
1590 		rcu_assign_pointer(d->domain->revmap[d->hwirq], d);  in irq_domain_fix_revmap()
1592 		slot = radix_tree_lookup_slot(&d->domain->revmap_tree, d->hwirq);  in irq_domain_fix_revmap()
1594 			radix_tree_replace_slot(&d->domain->revmap_tree, slot, d);  in irq_domain_fix_revmap()
1596 	mutex_unlock(&d->domain->revmap_mutex);  in irq_domain_fix_revmap()
1610 int irq_domain_push_irq(struct irq_domain *domain, int virq, void *arg)  in irq_domain_push_irq()  argument
1632 	if (domain == NULL)  in irq_domain_push_irq()
1635 	if (WARN_ON(!irq_domain_is_hierarchy(domain)))  in irq_domain_push_irq()
1641 	if (domain->parent != root_irq_data->domain)  in irq_domain_push_irq()
1659 	root_irq_data->domain = domain;  in irq_domain_push_irq()
1666 	rv = irq_domain_alloc_irqs_hierarchy(domain, virq, 1, arg);  in irq_domain_push_irq()
1675 	irq_domain_set_mapping(domain, root_irq_data->hwirq, root_irq_data);  in irq_domain_push_irq()
1692 int irq_domain_pop_irq(struct irq_domain *domain, int virq)  in irq_domain_pop_irq()  argument
1714 	if (domain == NULL)  in irq_domain_pop_irq()
1720 	tmp_irq_data = irq_domain_get_irq_data(domain, virq);  in irq_domain_pop_irq()
1726 	if (WARN_ON(root_irq_data->domain != domain))  in irq_domain_pop_irq()
1737 	irq_domain_clear_mapping(domain, root_irq_data->hwirq);  in irq_domain_pop_irq()
1738 	irq_domain_free_irqs_hierarchy(domain, virq, 1);  in irq_domain_pop_irq()
1763 	if (WARN(!data || !data->domain || !data->domain->ops->free,  in irq_domain_free_irqs()
1770 	irq_domain_free_irqs_hierarchy(data->domain, virq, nr_irqs);  in irq_domain_free_irqs()
1784 int irq_domain_alloc_irqs_parent(struct irq_domain *domain,  in irq_domain_alloc_irqs_parent()  argument
1788 	if (!domain->parent)  in irq_domain_alloc_irqs_parent()
1791 	return irq_domain_alloc_irqs_hierarchy(domain->parent, irq_base,  in irq_domain_alloc_irqs_parent()
1802 void irq_domain_free_irqs_parent(struct irq_domain *domain,  in irq_domain_free_irqs_parent()  argument
1805 	if (!domain->parent)  in irq_domain_free_irqs_parent()
1808 	irq_domain_free_irqs_hierarchy(domain->parent, irq_base, nr_irqs);  in irq_domain_free_irqs_parent()
1814 	if (irq_data && irq_data->domain) {  in __irq_domain_deactivate_irq()
1815 		struct irq_domain *domain = irq_data->domain;  in __irq_domain_deactivate_irq()  local
1817 		if (domain->ops->deactivate)  in __irq_domain_deactivate_irq()
1818 			domain->ops->deactivate(domain, irq_data);  in __irq_domain_deactivate_irq()
1828 	if (irqd && irqd->domain) {  in __irq_domain_activate_irq()
1829 		struct irq_domain *domain = irqd->domain;  in __irq_domain_activate_irq()  local
1834 		if (!ret && domain->ops->activate) {  in __irq_domain_activate_irq()
1835 			ret = domain->ops->activate(domain, irqd, reserve);  in __irq_domain_activate_irq()
1880 static void irq_domain_check_hierarchy(struct irq_domain *domain)  in irq_domain_check_hierarchy()  argument
1883 	if (domain->ops->alloc)  in irq_domain_check_hierarchy()
1884 		domain->flags |= IRQ_DOMAIN_FLAG_HIERARCHY;  in irq_domain_check_hierarchy()
1892 bool irq_domain_hierarchical_is_msi_remap(struct irq_domain *domain)  in irq_domain_hierarchical_is_msi_remap()  argument
1894 	for (; domain; domain = domain->parent) {  in irq_domain_hierarchical_is_msi_remap()
1895 		if (irq_domain_is_msi_remap(domain))  in irq_domain_hierarchical_is_msi_remap()
1906 struct irq_data *irq_domain_get_irq_data(struct irq_domain *domain,  in irq_domain_get_irq_data()  argument
1911 	return (irq_data && irq_data->domain == domain) ? irq_data : NULL;  in irq_domain_get_irq_data()
1926 void irq_domain_set_info(struct irq_domain *domain, unsigned int virq,  in irq_domain_set_info()  argument
1936 static int irq_domain_alloc_irqs_locked(struct irq_domain *domain, int irq_base,  in irq_domain_alloc_irqs_locked()  argument
1943 static void irq_domain_check_hierarchy(struct irq_domain *domain)  in irq_domain_check_hierarchy()  argument