• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Copyright (C) 1992, 1998-2006 Linus Torvalds, Ingo Molnar
3  * Copyright (C) 2005-2006, Thomas Gleixner, Russell King
4  *
5  * This file contains the interrupt descriptor management code
6  *
7  * Detailed information is available in Documentation/DocBook/genericirq
8  *
9  */
10 #include <linux/irq.h>
11 #include <linux/slab.h>
12 #include <linux/export.h>
13 #include <linux/interrupt.h>
14 #include <linux/kernel_stat.h>
15 #include <linux/radix-tree.h>
16 #include <linux/bitmap.h>
17 #include <linux/irqdomain.h>
18 
19 #include "internals.h"
20 
21 /*
22  * lockdep: we want to handle all irq_desc locks as a single lock-class:
23  */
24 static struct lock_class_key irq_desc_lock_class;
25 
26 #if defined(CONFIG_SMP)
init_irq_default_affinity(void)27 static void __init init_irq_default_affinity(void)
28 {
29 	alloc_cpumask_var(&irq_default_affinity, GFP_NOWAIT);
30 	cpumask_setall(irq_default_affinity);
31 }
32 #else
init_irq_default_affinity(void)33 static void __init init_irq_default_affinity(void)
34 {
35 }
36 #endif
37 
38 #ifdef CONFIG_SMP
alloc_masks(struct irq_desc * desc,gfp_t gfp,int node)39 static int alloc_masks(struct irq_desc *desc, gfp_t gfp, int node)
40 {
41 	if (!zalloc_cpumask_var_node(&desc->irq_common_data.affinity,
42 				     gfp, node))
43 		return -ENOMEM;
44 
45 #ifdef CONFIG_GENERIC_PENDING_IRQ
46 	if (!zalloc_cpumask_var_node(&desc->pending_mask, gfp, node)) {
47 		free_cpumask_var(desc->irq_common_data.affinity);
48 		return -ENOMEM;
49 	}
50 #endif
51 	return 0;
52 }
53 
desc_smp_init(struct irq_desc * desc,int node)54 static void desc_smp_init(struct irq_desc *desc, int node)
55 {
56 	cpumask_copy(desc->irq_common_data.affinity, irq_default_affinity);
57 #ifdef CONFIG_GENERIC_PENDING_IRQ
58 	cpumask_clear(desc->pending_mask);
59 #endif
60 #ifdef CONFIG_NUMA
61 	desc->irq_common_data.node = node;
62 #endif
63 }
64 
65 #else
66 static inline int
alloc_masks(struct irq_desc * desc,gfp_t gfp,int node)67 alloc_masks(struct irq_desc *desc, gfp_t gfp, int node) { return 0; }
desc_smp_init(struct irq_desc * desc,int node)68 static inline void desc_smp_init(struct irq_desc *desc, int node) { }
69 #endif
70 
desc_set_defaults(unsigned int irq,struct irq_desc * desc,int node,struct module * owner)71 static void desc_set_defaults(unsigned int irq, struct irq_desc *desc, int node,
72 		struct module *owner)
73 {
74 	int cpu;
75 
76 	desc->irq_common_data.handler_data = NULL;
77 	desc->irq_common_data.msi_desc = NULL;
78 
79 	desc->irq_data.common = &desc->irq_common_data;
80 	desc->irq_data.irq = irq;
81 	desc->irq_data.chip = &no_irq_chip;
82 	desc->irq_data.chip_data = NULL;
83 	irq_settings_clr_and_set(desc, ~0, _IRQ_DEFAULT_INIT_FLAGS);
84 	irqd_set(&desc->irq_data, IRQD_IRQ_DISABLED);
85 	desc->handle_irq = handle_bad_irq;
86 	desc->depth = 1;
87 	desc->irq_count = 0;
88 	desc->irqs_unhandled = 0;
89 	desc->name = NULL;
90 	desc->owner = owner;
91 	for_each_possible_cpu(cpu)
92 		*per_cpu_ptr(desc->kstat_irqs, cpu) = 0;
93 	desc_smp_init(desc, node);
94 }
95 
96 int nr_irqs = NR_IRQS;
97 EXPORT_SYMBOL_GPL(nr_irqs);
98 
99 static DEFINE_MUTEX(sparse_irq_lock);
100 static DECLARE_BITMAP(allocated_irqs, IRQ_BITMAP_BITS);
101 
102 #ifdef CONFIG_SPARSE_IRQ
103 
104 static RADIX_TREE(irq_desc_tree, GFP_KERNEL);
105 
irq_insert_desc(unsigned int irq,struct irq_desc * desc)106 static void irq_insert_desc(unsigned int irq, struct irq_desc *desc)
107 {
108 	radix_tree_insert(&irq_desc_tree, irq, desc);
109 }
110 
irq_to_desc(unsigned int irq)111 struct irq_desc *irq_to_desc(unsigned int irq)
112 {
113 	return radix_tree_lookup(&irq_desc_tree, irq);
114 }
115 EXPORT_SYMBOL(irq_to_desc);
116 
delete_irq_desc(unsigned int irq)117 static void delete_irq_desc(unsigned int irq)
118 {
119 	radix_tree_delete(&irq_desc_tree, irq);
120 }
121 
122 #ifdef CONFIG_SMP
free_masks(struct irq_desc * desc)123 static void free_masks(struct irq_desc *desc)
124 {
125 #ifdef CONFIG_GENERIC_PENDING_IRQ
126 	free_cpumask_var(desc->pending_mask);
127 #endif
128 	free_cpumask_var(desc->irq_common_data.affinity);
129 }
130 #else
free_masks(struct irq_desc * desc)131 static inline void free_masks(struct irq_desc *desc) { }
132 #endif
133 
irq_lock_sparse(void)134 void irq_lock_sparse(void)
135 {
136 	mutex_lock(&sparse_irq_lock);
137 }
138 
irq_unlock_sparse(void)139 void irq_unlock_sparse(void)
140 {
141 	mutex_unlock(&sparse_irq_lock);
142 }
143 
alloc_desc(int irq,int node,struct module * owner)144 static struct irq_desc *alloc_desc(int irq, int node, struct module *owner)
145 {
146 	struct irq_desc *desc;
147 	gfp_t gfp = GFP_KERNEL;
148 
149 	desc = kzalloc_node(sizeof(*desc), gfp, node);
150 	if (!desc)
151 		return NULL;
152 	/* allocate based on nr_cpu_ids */
153 	desc->kstat_irqs = alloc_percpu(unsigned int);
154 	if (!desc->kstat_irqs)
155 		goto err_desc;
156 
157 	if (alloc_masks(desc, gfp, node))
158 		goto err_kstat;
159 
160 	raw_spin_lock_init(&desc->lock);
161 	lockdep_set_class(&desc->lock, &irq_desc_lock_class);
162 
163 	desc_set_defaults(irq, desc, node, owner);
164 
165 	return desc;
166 
167 err_kstat:
168 	free_percpu(desc->kstat_irqs);
169 err_desc:
170 	kfree(desc);
171 	return NULL;
172 }
173 
free_desc(unsigned int irq)174 static void free_desc(unsigned int irq)
175 {
176 	struct irq_desc *desc = irq_to_desc(irq);
177 
178 	unregister_irq_proc(irq, desc);
179 
180 	/*
181 	 * sparse_irq_lock protects also show_interrupts() and
182 	 * kstat_irq_usr(). Once we deleted the descriptor from the
183 	 * sparse tree we can free it. Access in proc will fail to
184 	 * lookup the descriptor.
185 	 */
186 	mutex_lock(&sparse_irq_lock);
187 	delete_irq_desc(irq);
188 	mutex_unlock(&sparse_irq_lock);
189 
190 	free_masks(desc);
191 	free_percpu(desc->kstat_irqs);
192 	kfree(desc);
193 }
194 
alloc_descs(unsigned int start,unsigned int cnt,int node,struct module * owner)195 static int alloc_descs(unsigned int start, unsigned int cnt, int node,
196 		       struct module *owner)
197 {
198 	struct irq_desc *desc;
199 	int i;
200 
201 	for (i = 0; i < cnt; i++) {
202 		desc = alloc_desc(start + i, node, owner);
203 		if (!desc)
204 			goto err;
205 		mutex_lock(&sparse_irq_lock);
206 		irq_insert_desc(start + i, desc);
207 		mutex_unlock(&sparse_irq_lock);
208 	}
209 	return start;
210 
211 err:
212 	for (i--; i >= 0; i--)
213 		free_desc(start + i);
214 
215 	mutex_lock(&sparse_irq_lock);
216 	bitmap_clear(allocated_irqs, start, cnt);
217 	mutex_unlock(&sparse_irq_lock);
218 	return -ENOMEM;
219 }
220 
irq_expand_nr_irqs(unsigned int nr)221 static int irq_expand_nr_irqs(unsigned int nr)
222 {
223 	if (nr > IRQ_BITMAP_BITS)
224 		return -ENOMEM;
225 	nr_irqs = nr;
226 	return 0;
227 }
228 
early_irq_init(void)229 int __init early_irq_init(void)
230 {
231 	int i, initcnt, node = first_online_node;
232 	struct irq_desc *desc;
233 
234 	init_irq_default_affinity();
235 
236 	/* Let arch update nr_irqs and return the nr of preallocated irqs */
237 	initcnt = arch_probe_nr_irqs();
238 	printk(KERN_INFO "NR_IRQS:%d nr_irqs:%d %d\n", NR_IRQS, nr_irqs, initcnt);
239 
240 	if (WARN_ON(nr_irqs > IRQ_BITMAP_BITS))
241 		nr_irqs = IRQ_BITMAP_BITS;
242 
243 	if (WARN_ON(initcnt > IRQ_BITMAP_BITS))
244 		initcnt = IRQ_BITMAP_BITS;
245 
246 	if (initcnt > nr_irqs)
247 		nr_irqs = initcnt;
248 
249 	for (i = 0; i < initcnt; i++) {
250 		desc = alloc_desc(i, node, NULL);
251 		set_bit(i, allocated_irqs);
252 		irq_insert_desc(i, desc);
253 	}
254 	return arch_early_irq_init();
255 }
256 
257 #else /* !CONFIG_SPARSE_IRQ */
258 
259 struct irq_desc irq_desc[NR_IRQS] __cacheline_aligned_in_smp = {
260 	[0 ... NR_IRQS-1] = {
261 		.handle_irq	= handle_bad_irq,
262 		.depth		= 1,
263 		.lock		= __RAW_SPIN_LOCK_UNLOCKED(irq_desc->lock),
264 	}
265 };
266 
early_irq_init(void)267 int __init early_irq_init(void)
268 {
269 	int count, i, node = first_online_node;
270 	struct irq_desc *desc;
271 
272 	init_irq_default_affinity();
273 
274 	printk(KERN_INFO "NR_IRQS:%d\n", NR_IRQS);
275 
276 	desc = irq_desc;
277 	count = ARRAY_SIZE(irq_desc);
278 
279 	for (i = 0; i < count; i++) {
280 		desc[i].kstat_irqs = alloc_percpu(unsigned int);
281 		alloc_masks(&desc[i], GFP_KERNEL, node);
282 		raw_spin_lock_init(&desc[i].lock);
283 		lockdep_set_class(&desc[i].lock, &irq_desc_lock_class);
284 		desc_set_defaults(i, &desc[i], node, NULL);
285 	}
286 	return arch_early_irq_init();
287 }
288 
irq_to_desc(unsigned int irq)289 struct irq_desc *irq_to_desc(unsigned int irq)
290 {
291 	return (irq < NR_IRQS) ? irq_desc + irq : NULL;
292 }
293 EXPORT_SYMBOL(irq_to_desc);
294 
free_desc(unsigned int irq)295 static void free_desc(unsigned int irq)
296 {
297 	struct irq_desc *desc = irq_to_desc(irq);
298 	unsigned long flags;
299 
300 	raw_spin_lock_irqsave(&desc->lock, flags);
301 	desc_set_defaults(irq, desc, irq_desc_get_node(desc), NULL);
302 	raw_spin_unlock_irqrestore(&desc->lock, flags);
303 }
304 
alloc_descs(unsigned int start,unsigned int cnt,int node,struct module * owner)305 static inline int alloc_descs(unsigned int start, unsigned int cnt, int node,
306 			      struct module *owner)
307 {
308 	u32 i;
309 
310 	for (i = 0; i < cnt; i++) {
311 		struct irq_desc *desc = irq_to_desc(start + i);
312 
313 		desc->owner = owner;
314 	}
315 	return start;
316 }
317 
irq_expand_nr_irqs(unsigned int nr)318 static int irq_expand_nr_irqs(unsigned int nr)
319 {
320 	return -ENOMEM;
321 }
322 
irq_mark_irq(unsigned int irq)323 void irq_mark_irq(unsigned int irq)
324 {
325 	mutex_lock(&sparse_irq_lock);
326 	bitmap_set(allocated_irqs, irq, 1);
327 	mutex_unlock(&sparse_irq_lock);
328 }
329 
330 #ifdef CONFIG_GENERIC_IRQ_LEGACY
irq_init_desc(unsigned int irq)331 void irq_init_desc(unsigned int irq)
332 {
333 	free_desc(irq);
334 }
335 #endif
336 
337 #endif /* !CONFIG_SPARSE_IRQ */
338 
339 /**
340  * generic_handle_irq - Invoke the handler for a particular irq
341  * @irq:	The irq number to handle
342  *
343  */
generic_handle_irq(unsigned int irq)344 int generic_handle_irq(unsigned int irq)
345 {
346 	struct irq_desc *desc = irq_to_desc(irq);
347 
348 	if (!desc)
349 		return -EINVAL;
350 	generic_handle_irq_desc(desc);
351 	return 0;
352 }
353 EXPORT_SYMBOL_GPL(generic_handle_irq);
354 
355 #ifdef CONFIG_HANDLE_DOMAIN_IRQ
356 /**
357  * __handle_domain_irq - Invoke the handler for a HW irq belonging to a domain
358  * @domain:	The domain where to perform the lookup
359  * @hwirq:	The HW irq number to convert to a logical one
360  * @lookup:	Whether to perform the domain lookup or not
361  * @regs:	Register file coming from the low-level handling code
362  *
363  * Returns:	0 on success, or -EINVAL if conversion has failed
364  */
__handle_domain_irq(struct irq_domain * domain,unsigned int hwirq,bool lookup,struct pt_regs * regs)365 int __handle_domain_irq(struct irq_domain *domain, unsigned int hwirq,
366 			bool lookup, struct pt_regs *regs)
367 {
368 	struct pt_regs *old_regs = set_irq_regs(regs);
369 	unsigned int irq = hwirq;
370 	int ret = 0;
371 
372 	irq_enter();
373 
374 #ifdef CONFIG_IRQ_DOMAIN
375 	if (lookup)
376 		irq = irq_find_mapping(domain, hwirq);
377 #endif
378 
379 	/*
380 	 * Some hardware gives randomly wrong interrupts.  Rather
381 	 * than crashing, do something sensible.
382 	 */
383 	if (unlikely(!irq || irq >= nr_irqs)) {
384 		ack_bad_irq(irq);
385 		ret = -EINVAL;
386 	} else {
387 		generic_handle_irq(irq);
388 	}
389 
390 	irq_exit();
391 	set_irq_regs(old_regs);
392 	return ret;
393 }
394 #endif
395 
396 /* Dynamic interrupt handling */
397 
398 /**
399  * irq_free_descs - free irq descriptors
400  * @from:	Start of descriptor range
401  * @cnt:	Number of consecutive irqs to free
402  */
irq_free_descs(unsigned int from,unsigned int cnt)403 void irq_free_descs(unsigned int from, unsigned int cnt)
404 {
405 	int i;
406 
407 	if (from >= nr_irqs || (from + cnt) > nr_irqs)
408 		return;
409 
410 	for (i = 0; i < cnt; i++)
411 		free_desc(from + i);
412 
413 	mutex_lock(&sparse_irq_lock);
414 	bitmap_clear(allocated_irqs, from, cnt);
415 	mutex_unlock(&sparse_irq_lock);
416 }
417 EXPORT_SYMBOL_GPL(irq_free_descs);
418 
419 /**
420  * irq_alloc_descs - allocate and initialize a range of irq descriptors
421  * @irq:	Allocate for specific irq number if irq >= 0
422  * @from:	Start the search from this irq number
423  * @cnt:	Number of consecutive irqs to allocate.
424  * @node:	Preferred node on which the irq descriptor should be allocated
425  * @owner:	Owning module (can be NULL)
426  *
427  * Returns the first irq number or error code
428  */
429 int __ref
__irq_alloc_descs(int irq,unsigned int from,unsigned int cnt,int node,struct module * owner)430 __irq_alloc_descs(int irq, unsigned int from, unsigned int cnt, int node,
431 		  struct module *owner)
432 {
433 	int start, ret;
434 
435 	if (!cnt)
436 		return -EINVAL;
437 
438 	if (irq >= 0) {
439 		if (from > irq)
440 			return -EINVAL;
441 		from = irq;
442 	} else {
443 		/*
444 		 * For interrupts which are freely allocated the
445 		 * architecture can force a lower bound to the @from
446 		 * argument. x86 uses this to exclude the GSI space.
447 		 */
448 		from = arch_dynirq_lower_bound(from);
449 	}
450 
451 	mutex_lock(&sparse_irq_lock);
452 
453 	start = bitmap_find_next_zero_area(allocated_irqs, IRQ_BITMAP_BITS,
454 					   from, cnt, 0);
455 	ret = -EEXIST;
456 	if (irq >=0 && start != irq)
457 		goto err;
458 
459 	if (start + cnt > nr_irqs) {
460 		ret = irq_expand_nr_irqs(start + cnt);
461 		if (ret)
462 			goto err;
463 	}
464 
465 	bitmap_set(allocated_irqs, start, cnt);
466 	mutex_unlock(&sparse_irq_lock);
467 	return alloc_descs(start, cnt, node, owner);
468 
469 err:
470 	mutex_unlock(&sparse_irq_lock);
471 	return ret;
472 }
473 EXPORT_SYMBOL_GPL(__irq_alloc_descs);
474 
475 #ifdef CONFIG_GENERIC_IRQ_LEGACY_ALLOC_HWIRQ
476 /**
477  * irq_alloc_hwirqs - Allocate an irq descriptor and initialize the hardware
478  * @cnt:	number of interrupts to allocate
479  * @node:	node on which to allocate
480  *
481  * Returns an interrupt number > 0 or 0, if the allocation fails.
482  */
irq_alloc_hwirqs(int cnt,int node)483 unsigned int irq_alloc_hwirqs(int cnt, int node)
484 {
485 	int i, irq = __irq_alloc_descs(-1, 0, cnt, node, NULL);
486 
487 	if (irq < 0)
488 		return 0;
489 
490 	for (i = irq; cnt > 0; i++, cnt--) {
491 		if (arch_setup_hwirq(i, node))
492 			goto err;
493 		irq_clear_status_flags(i, _IRQ_NOREQUEST);
494 	}
495 	return irq;
496 
497 err:
498 	for (i--; i >= irq; i--) {
499 		irq_set_status_flags(i, _IRQ_NOREQUEST | _IRQ_NOPROBE);
500 		arch_teardown_hwirq(i);
501 	}
502 	irq_free_descs(irq, cnt);
503 	return 0;
504 }
505 EXPORT_SYMBOL_GPL(irq_alloc_hwirqs);
506 
507 /**
508  * irq_free_hwirqs - Free irq descriptor and cleanup the hardware
509  * @from:	Free from irq number
510  * @cnt:	number of interrupts to free
511  *
512  */
irq_free_hwirqs(unsigned int from,int cnt)513 void irq_free_hwirqs(unsigned int from, int cnt)
514 {
515 	int i, j;
516 
517 	for (i = from, j = cnt; j > 0; i++, j--) {
518 		irq_set_status_flags(i, _IRQ_NOREQUEST | _IRQ_NOPROBE);
519 		arch_teardown_hwirq(i);
520 	}
521 	irq_free_descs(from, cnt);
522 }
523 EXPORT_SYMBOL_GPL(irq_free_hwirqs);
524 #endif
525 
526 /**
527  * irq_get_next_irq - get next allocated irq number
528  * @offset:	where to start the search
529  *
530  * Returns next irq number after offset or nr_irqs if none is found.
531  */
irq_get_next_irq(unsigned int offset)532 unsigned int irq_get_next_irq(unsigned int offset)
533 {
534 	return find_next_bit(allocated_irqs, nr_irqs, offset);
535 }
536 
537 struct irq_desc *
__irq_get_desc_lock(unsigned int irq,unsigned long * flags,bool bus,unsigned int check)538 __irq_get_desc_lock(unsigned int irq, unsigned long *flags, bool bus,
539 		    unsigned int check)
540 {
541 	struct irq_desc *desc = irq_to_desc(irq);
542 
543 	if (desc) {
544 		if (check & _IRQ_DESC_CHECK) {
545 			if ((check & _IRQ_DESC_PERCPU) &&
546 			    !irq_settings_is_per_cpu_devid(desc))
547 				return NULL;
548 
549 			if (!(check & _IRQ_DESC_PERCPU) &&
550 			    irq_settings_is_per_cpu_devid(desc))
551 				return NULL;
552 		}
553 
554 		if (bus)
555 			chip_bus_lock(desc);
556 		raw_spin_lock_irqsave(&desc->lock, *flags);
557 	}
558 	return desc;
559 }
560 
__irq_put_desc_unlock(struct irq_desc * desc,unsigned long flags,bool bus)561 void __irq_put_desc_unlock(struct irq_desc *desc, unsigned long flags, bool bus)
562 {
563 	raw_spin_unlock_irqrestore(&desc->lock, flags);
564 	if (bus)
565 		chip_bus_sync_unlock(desc);
566 }
567 
irq_set_percpu_devid(unsigned int irq)568 int irq_set_percpu_devid(unsigned int irq)
569 {
570 	struct irq_desc *desc = irq_to_desc(irq);
571 
572 	if (!desc)
573 		return -EINVAL;
574 
575 	if (desc->percpu_enabled)
576 		return -EINVAL;
577 
578 	desc->percpu_enabled = kzalloc(sizeof(*desc->percpu_enabled), GFP_KERNEL);
579 
580 	if (!desc->percpu_enabled)
581 		return -ENOMEM;
582 
583 	irq_set_percpu_devid_flags(irq);
584 	return 0;
585 }
586 
kstat_incr_irq_this_cpu(unsigned int irq)587 void kstat_incr_irq_this_cpu(unsigned int irq)
588 {
589 	kstat_incr_irqs_this_cpu(irq_to_desc(irq));
590 }
591 
592 /**
593  * kstat_irqs_cpu - Get the statistics for an interrupt on a cpu
594  * @irq:	The interrupt number
595  * @cpu:	The cpu number
596  *
597  * Returns the sum of interrupt counts on @cpu since boot for
598  * @irq. The caller must ensure that the interrupt is not removed
599  * concurrently.
600  */
kstat_irqs_cpu(unsigned int irq,int cpu)601 unsigned int kstat_irqs_cpu(unsigned int irq, int cpu)
602 {
603 	struct irq_desc *desc = irq_to_desc(irq);
604 
605 	return desc && desc->kstat_irqs ?
606 			*per_cpu_ptr(desc->kstat_irqs, cpu) : 0;
607 }
608 
609 /**
610  * kstat_irqs - Get the statistics for an interrupt
611  * @irq:	The interrupt number
612  *
613  * Returns the sum of interrupt counts on all cpus since boot for
614  * @irq. The caller must ensure that the interrupt is not removed
615  * concurrently.
616  */
kstat_irqs(unsigned int irq)617 unsigned int kstat_irqs(unsigned int irq)
618 {
619 	struct irq_desc *desc = irq_to_desc(irq);
620 	int cpu;
621 	unsigned int sum = 0;
622 
623 	if (!desc || !desc->kstat_irqs)
624 		return 0;
625 	for_each_possible_cpu(cpu)
626 		sum += *per_cpu_ptr(desc->kstat_irqs, cpu);
627 	return sum;
628 }
629 
630 /**
631  * kstat_irqs_usr - Get the statistics for an interrupt
632  * @irq:	The interrupt number
633  *
634  * Returns the sum of interrupt counts on all cpus since boot for
635  * @irq. Contrary to kstat_irqs() this can be called from any
636  * preemptible context. It's protected against concurrent removal of
637  * an interrupt descriptor when sparse irqs are enabled.
638  */
kstat_irqs_usr(unsigned int irq)639 unsigned int kstat_irqs_usr(unsigned int irq)
640 {
641 	unsigned int sum;
642 
643 	irq_lock_sparse();
644 	sum = kstat_irqs(irq);
645 	irq_unlock_sparse();
646 	return sum;
647 }
648