• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * linux/kernel/irq/chip.c
3  *
4  * Copyright (C) 1992, 1998-2006 Linus Torvalds, Ingo Molnar
5  * Copyright (C) 2005-2006, Thomas Gleixner, Russell King
6  *
7  * This file contains the core interrupt handling code, for irq-chip
8  * based architectures.
9  *
10  * Detailed information is available in Documentation/DocBook/genericirq
11  */
12 
13 #include <linux/irq.h>
14 #include <linux/msi.h>
15 #include <linux/module.h>
16 #include <linux/interrupt.h>
17 #include <linux/kernel_stat.h>
18 #include <linux/irqdomain.h>
19 
20 #include <trace/events/irq.h>
21 
22 #include "internals.h"
23 
bad_chained_irq(int irq,void * dev_id)24 static irqreturn_t bad_chained_irq(int irq, void *dev_id)
25 {
26 	WARN_ONCE(1, "Chained irq %d should not call an action\n", irq);
27 	return IRQ_NONE;
28 }
29 
30 /*
31  * Chained handlers should never call action on their IRQ. This default
32  * action will emit warning if such thing happens.
33  */
34 struct irqaction chained_action = {
35 	.handler = bad_chained_irq,
36 };
37 
38 /**
39  *	irq_set_chip - set the irq chip for an irq
40  *	@irq:	irq number
41  *	@chip:	pointer to irq chip description structure
42  */
irq_set_chip(unsigned int irq,struct irq_chip * chip)43 int irq_set_chip(unsigned int irq, struct irq_chip *chip)
44 {
45 	unsigned long flags;
46 	struct irq_desc *desc = irq_get_desc_lock(irq, &flags, 0);
47 
48 	if (!desc)
49 		return -EINVAL;
50 
51 	if (!chip)
52 		chip = &no_irq_chip;
53 
54 	desc->irq_data.chip = chip;
55 	irq_put_desc_unlock(desc, flags);
56 	/*
57 	 * For !CONFIG_SPARSE_IRQ make the irq show up in
58 	 * allocated_irqs.
59 	 */
60 	irq_mark_irq(irq);
61 	return 0;
62 }
63 EXPORT_SYMBOL(irq_set_chip);
64 
65 /**
66  *	irq_set_type - set the irq trigger type for an irq
67  *	@irq:	irq number
68  *	@type:	IRQ_TYPE_{LEVEL,EDGE}_* value - see include/linux/irq.h
69  */
irq_set_irq_type(unsigned int irq,unsigned int type)70 int irq_set_irq_type(unsigned int irq, unsigned int type)
71 {
72 	unsigned long flags;
73 	struct irq_desc *desc = irq_get_desc_buslock(irq, &flags, IRQ_GET_DESC_CHECK_GLOBAL);
74 	int ret = 0;
75 
76 	if (!desc)
77 		return -EINVAL;
78 
79 	type &= IRQ_TYPE_SENSE_MASK;
80 	ret = __irq_set_trigger(desc, type);
81 	irq_put_desc_busunlock(desc, flags);
82 	return ret;
83 }
84 EXPORT_SYMBOL(irq_set_irq_type);
85 
86 /**
87  *	irq_set_handler_data - set irq handler data for an irq
88  *	@irq:	Interrupt number
89  *	@data:	Pointer to interrupt specific data
90  *
91  *	Set the hardware irq controller data for an irq
92  */
irq_set_handler_data(unsigned int irq,void * data)93 int irq_set_handler_data(unsigned int irq, void *data)
94 {
95 	unsigned long flags;
96 	struct irq_desc *desc = irq_get_desc_lock(irq, &flags, 0);
97 
98 	if (!desc)
99 		return -EINVAL;
100 	desc->irq_common_data.handler_data = data;
101 	irq_put_desc_unlock(desc, flags);
102 	return 0;
103 }
104 EXPORT_SYMBOL(irq_set_handler_data);
105 
106 /**
107  *	irq_set_msi_desc_off - set MSI descriptor data for an irq at offset
108  *	@irq_base:	Interrupt number base
109  *	@irq_offset:	Interrupt number offset
110  *	@entry:		Pointer to MSI descriptor data
111  *
112  *	Set the MSI descriptor entry for an irq at offset
113  */
irq_set_msi_desc_off(unsigned int irq_base,unsigned int irq_offset,struct msi_desc * entry)114 int irq_set_msi_desc_off(unsigned int irq_base, unsigned int irq_offset,
115 			 struct msi_desc *entry)
116 {
117 	unsigned long flags;
118 	struct irq_desc *desc = irq_get_desc_lock(irq_base + irq_offset, &flags, IRQ_GET_DESC_CHECK_GLOBAL);
119 
120 	if (!desc)
121 		return -EINVAL;
122 	desc->irq_common_data.msi_desc = entry;
123 	if (entry && !irq_offset)
124 		entry->irq = irq_base;
125 	irq_put_desc_unlock(desc, flags);
126 	return 0;
127 }
128 
129 /**
130  *	irq_set_msi_desc - set MSI descriptor data for an irq
131  *	@irq:	Interrupt number
132  *	@entry:	Pointer to MSI descriptor data
133  *
134  *	Set the MSI descriptor entry for an irq
135  */
irq_set_msi_desc(unsigned int irq,struct msi_desc * entry)136 int irq_set_msi_desc(unsigned int irq, struct msi_desc *entry)
137 {
138 	return irq_set_msi_desc_off(irq, 0, entry);
139 }
140 
141 /**
142  *	irq_set_chip_data - set irq chip data for an irq
143  *	@irq:	Interrupt number
144  *	@data:	Pointer to chip specific data
145  *
146  *	Set the hardware irq chip data for an irq
147  */
irq_set_chip_data(unsigned int irq,void * data)148 int irq_set_chip_data(unsigned int irq, void *data)
149 {
150 	unsigned long flags;
151 	struct irq_desc *desc = irq_get_desc_lock(irq, &flags, 0);
152 
153 	if (!desc)
154 		return -EINVAL;
155 	desc->irq_data.chip_data = data;
156 	irq_put_desc_unlock(desc, flags);
157 	return 0;
158 }
159 EXPORT_SYMBOL(irq_set_chip_data);
160 
irq_get_irq_data(unsigned int irq)161 struct irq_data *irq_get_irq_data(unsigned int irq)
162 {
163 	struct irq_desc *desc = irq_to_desc(irq);
164 
165 	return desc ? &desc->irq_data : NULL;
166 }
167 EXPORT_SYMBOL_GPL(irq_get_irq_data);
168 
irq_state_clr_disabled(struct irq_desc * desc)169 static void irq_state_clr_disabled(struct irq_desc *desc)
170 {
171 	irqd_clear(&desc->irq_data, IRQD_IRQ_DISABLED);
172 }
173 
irq_state_set_disabled(struct irq_desc * desc)174 static void irq_state_set_disabled(struct irq_desc *desc)
175 {
176 	irqd_set(&desc->irq_data, IRQD_IRQ_DISABLED);
177 }
178 
irq_state_clr_masked(struct irq_desc * desc)179 static void irq_state_clr_masked(struct irq_desc *desc)
180 {
181 	irqd_clear(&desc->irq_data, IRQD_IRQ_MASKED);
182 }
183 
irq_state_set_masked(struct irq_desc * desc)184 static void irq_state_set_masked(struct irq_desc *desc)
185 {
186 	irqd_set(&desc->irq_data, IRQD_IRQ_MASKED);
187 }
188 
irq_startup(struct irq_desc * desc,bool resend)189 int irq_startup(struct irq_desc *desc, bool resend)
190 {
191 	int ret = 0;
192 
193 	irq_state_clr_disabled(desc);
194 	desc->depth = 0;
195 
196 	irq_domain_activate_irq(&desc->irq_data);
197 	if (desc->irq_data.chip->irq_startup) {
198 		ret = desc->irq_data.chip->irq_startup(&desc->irq_data);
199 		irq_state_clr_masked(desc);
200 	} else {
201 		irq_enable(desc);
202 	}
203 	if (resend)
204 		check_irq_resend(desc);
205 	return ret;
206 }
207 
irq_shutdown(struct irq_desc * desc)208 void irq_shutdown(struct irq_desc *desc)
209 {
210 	irq_state_set_disabled(desc);
211 	desc->depth = 1;
212 	if (desc->irq_data.chip->irq_shutdown)
213 		desc->irq_data.chip->irq_shutdown(&desc->irq_data);
214 	else if (desc->irq_data.chip->irq_disable)
215 		desc->irq_data.chip->irq_disable(&desc->irq_data);
216 	else
217 		desc->irq_data.chip->irq_mask(&desc->irq_data);
218 	irq_domain_deactivate_irq(&desc->irq_data);
219 	irq_state_set_masked(desc);
220 }
221 
irq_enable(struct irq_desc * desc)222 void irq_enable(struct irq_desc *desc)
223 {
224 	irq_state_clr_disabled(desc);
225 	if (desc->irq_data.chip->irq_enable)
226 		desc->irq_data.chip->irq_enable(&desc->irq_data);
227 	else
228 		desc->irq_data.chip->irq_unmask(&desc->irq_data);
229 	irq_state_clr_masked(desc);
230 }
231 
232 /**
233  * irq_disable - Mark interrupt disabled
234  * @desc:	irq descriptor which should be disabled
235  *
236  * If the chip does not implement the irq_disable callback, we
237  * use a lazy disable approach. That means we mark the interrupt
238  * disabled, but leave the hardware unmasked. That's an
239  * optimization because we avoid the hardware access for the
240  * common case where no interrupt happens after we marked it
241  * disabled. If an interrupt happens, then the interrupt flow
242  * handler masks the line at the hardware level and marks it
243  * pending.
244  *
245  * If the interrupt chip does not implement the irq_disable callback,
246  * a driver can disable the lazy approach for a particular irq line by
247  * calling 'irq_set_status_flags(irq, IRQ_DISABLE_UNLAZY)'. This can
248  * be used for devices which cannot disable the interrupt at the
249  * device level under certain circumstances and have to use
250  * disable_irq[_nosync] instead.
251  */
irq_disable(struct irq_desc * desc)252 void irq_disable(struct irq_desc *desc)
253 {
254 	irq_state_set_disabled(desc);
255 	if (desc->irq_data.chip->irq_disable) {
256 		desc->irq_data.chip->irq_disable(&desc->irq_data);
257 		irq_state_set_masked(desc);
258 	} else if (irq_settings_disable_unlazy(desc)) {
259 		mask_irq(desc);
260 	}
261 }
262 
irq_percpu_enable(struct irq_desc * desc,unsigned int cpu)263 void irq_percpu_enable(struct irq_desc *desc, unsigned int cpu)
264 {
265 	if (desc->irq_data.chip->irq_enable)
266 		desc->irq_data.chip->irq_enable(&desc->irq_data);
267 	else
268 		desc->irq_data.chip->irq_unmask(&desc->irq_data);
269 	cpumask_set_cpu(cpu, desc->percpu_enabled);
270 }
271 
irq_percpu_disable(struct irq_desc * desc,unsigned int cpu)272 void irq_percpu_disable(struct irq_desc *desc, unsigned int cpu)
273 {
274 	if (desc->irq_data.chip->irq_disable)
275 		desc->irq_data.chip->irq_disable(&desc->irq_data);
276 	else
277 		desc->irq_data.chip->irq_mask(&desc->irq_data);
278 	cpumask_clear_cpu(cpu, desc->percpu_enabled);
279 }
280 
mask_ack_irq(struct irq_desc * desc)281 static inline void mask_ack_irq(struct irq_desc *desc)
282 {
283 	if (desc->irq_data.chip->irq_mask_ack)
284 		desc->irq_data.chip->irq_mask_ack(&desc->irq_data);
285 	else {
286 		desc->irq_data.chip->irq_mask(&desc->irq_data);
287 		if (desc->irq_data.chip->irq_ack)
288 			desc->irq_data.chip->irq_ack(&desc->irq_data);
289 	}
290 	irq_state_set_masked(desc);
291 }
292 
mask_irq(struct irq_desc * desc)293 void mask_irq(struct irq_desc *desc)
294 {
295 	if (desc->irq_data.chip->irq_mask) {
296 		desc->irq_data.chip->irq_mask(&desc->irq_data);
297 		irq_state_set_masked(desc);
298 	}
299 }
300 
unmask_irq(struct irq_desc * desc)301 void unmask_irq(struct irq_desc *desc)
302 {
303 	if (desc->irq_data.chip->irq_unmask) {
304 		desc->irq_data.chip->irq_unmask(&desc->irq_data);
305 		irq_state_clr_masked(desc);
306 	}
307 }
308 
unmask_threaded_irq(struct irq_desc * desc)309 void unmask_threaded_irq(struct irq_desc *desc)
310 {
311 	struct irq_chip *chip = desc->irq_data.chip;
312 
313 	if (chip->flags & IRQCHIP_EOI_THREADED)
314 		chip->irq_eoi(&desc->irq_data);
315 
316 	if (chip->irq_unmask) {
317 		chip->irq_unmask(&desc->irq_data);
318 		irq_state_clr_masked(desc);
319 	}
320 }
321 
322 /*
323  *	handle_nested_irq - Handle a nested irq from a irq thread
324  *	@irq:	the interrupt number
325  *
326  *	Handle interrupts which are nested into a threaded interrupt
327  *	handler. The handler function is called inside the calling
328  *	threads context.
329  */
handle_nested_irq(unsigned int irq)330 void handle_nested_irq(unsigned int irq)
331 {
332 	struct irq_desc *desc = irq_to_desc(irq);
333 	struct irqaction *action;
334 	irqreturn_t action_ret;
335 
336 	might_sleep();
337 
338 	raw_spin_lock_irq(&desc->lock);
339 
340 	desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
341 
342 	action = desc->action;
343 	if (unlikely(!action || irqd_irq_disabled(&desc->irq_data))) {
344 		desc->istate |= IRQS_PENDING;
345 		goto out_unlock;
346 	}
347 
348 	kstat_incr_irqs_this_cpu(desc);
349 	irqd_set(&desc->irq_data, IRQD_IRQ_INPROGRESS);
350 	raw_spin_unlock_irq(&desc->lock);
351 
352 	action_ret = action->thread_fn(action->irq, action->dev_id);
353 	if (!noirqdebug)
354 		note_interrupt(desc, action_ret);
355 
356 	raw_spin_lock_irq(&desc->lock);
357 	irqd_clear(&desc->irq_data, IRQD_IRQ_INPROGRESS);
358 
359 out_unlock:
360 	raw_spin_unlock_irq(&desc->lock);
361 }
362 EXPORT_SYMBOL_GPL(handle_nested_irq);
363 
irq_check_poll(struct irq_desc * desc)364 static bool irq_check_poll(struct irq_desc *desc)
365 {
366 	if (!(desc->istate & IRQS_POLL_INPROGRESS))
367 		return false;
368 	return irq_wait_for_poll(desc);
369 }
370 
irq_may_run(struct irq_desc * desc)371 static bool irq_may_run(struct irq_desc *desc)
372 {
373 	unsigned int mask = IRQD_IRQ_INPROGRESS | IRQD_WAKEUP_ARMED;
374 
375 	/*
376 	 * If the interrupt is not in progress and is not an armed
377 	 * wakeup interrupt, proceed.
378 	 */
379 	if (!irqd_has_set(&desc->irq_data, mask))
380 		return true;
381 
382 	/*
383 	 * If the interrupt is an armed wakeup source, mark it pending
384 	 * and suspended, disable it and notify the pm core about the
385 	 * event.
386 	 */
387 	if (irq_pm_check_wakeup(desc))
388 		return false;
389 
390 	/*
391 	 * Handle a potential concurrent poll on a different core.
392 	 */
393 	return irq_check_poll(desc);
394 }
395 
396 /**
397  *	handle_simple_irq - Simple and software-decoded IRQs.
398  *	@desc:	the interrupt description structure for this irq
399  *
400  *	Simple interrupts are either sent from a demultiplexing interrupt
401  *	handler or come from hardware, where no interrupt hardware control
402  *	is necessary.
403  *
404  *	Note: The caller is expected to handle the ack, clear, mask and
405  *	unmask issues if necessary.
406  */
handle_simple_irq(struct irq_desc * desc)407 void handle_simple_irq(struct irq_desc *desc)
408 {
409 	raw_spin_lock(&desc->lock);
410 
411 	if (!irq_may_run(desc))
412 		goto out_unlock;
413 
414 	desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
415 
416 	if (unlikely(!desc->action || irqd_irq_disabled(&desc->irq_data))) {
417 		desc->istate |= IRQS_PENDING;
418 		goto out_unlock;
419 	}
420 
421 	kstat_incr_irqs_this_cpu(desc);
422 	handle_irq_event(desc);
423 
424 out_unlock:
425 	raw_spin_unlock(&desc->lock);
426 }
427 EXPORT_SYMBOL_GPL(handle_simple_irq);
428 
429 /*
430  * Called unconditionally from handle_level_irq() and only for oneshot
431  * interrupts from handle_fasteoi_irq()
432  */
cond_unmask_irq(struct irq_desc * desc)433 static void cond_unmask_irq(struct irq_desc *desc)
434 {
435 	/*
436 	 * We need to unmask in the following cases:
437 	 * - Standard level irq (IRQF_ONESHOT is not set)
438 	 * - Oneshot irq which did not wake the thread (caused by a
439 	 *   spurious interrupt or a primary handler handling it
440 	 *   completely).
441 	 */
442 	if (!irqd_irq_disabled(&desc->irq_data) &&
443 	    irqd_irq_masked(&desc->irq_data) && !desc->threads_oneshot)
444 		unmask_irq(desc);
445 }
446 
447 /**
448  *	handle_level_irq - Level type irq handler
449  *	@desc:	the interrupt description structure for this irq
450  *
451  *	Level type interrupts are active as long as the hardware line has
452  *	the active level. This may require to mask the interrupt and unmask
453  *	it after the associated handler has acknowledged the device, so the
454  *	interrupt line is back to inactive.
455  */
handle_level_irq(struct irq_desc * desc)456 void handle_level_irq(struct irq_desc *desc)
457 {
458 	raw_spin_lock(&desc->lock);
459 	mask_ack_irq(desc);
460 
461 	if (!irq_may_run(desc))
462 		goto out_unlock;
463 
464 	desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
465 
466 	/*
467 	 * If its disabled or no action available
468 	 * keep it masked and get out of here
469 	 */
470 	if (unlikely(!desc->action || irqd_irq_disabled(&desc->irq_data))) {
471 		desc->istate |= IRQS_PENDING;
472 		goto out_unlock;
473 	}
474 
475 	kstat_incr_irqs_this_cpu(desc);
476 	handle_irq_event(desc);
477 
478 	cond_unmask_irq(desc);
479 
480 out_unlock:
481 	raw_spin_unlock(&desc->lock);
482 }
483 EXPORT_SYMBOL_GPL(handle_level_irq);
484 
485 #ifdef CONFIG_IRQ_PREFLOW_FASTEOI
preflow_handler(struct irq_desc * desc)486 static inline void preflow_handler(struct irq_desc *desc)
487 {
488 	if (desc->preflow_handler)
489 		desc->preflow_handler(&desc->irq_data);
490 }
491 #else
preflow_handler(struct irq_desc * desc)492 static inline void preflow_handler(struct irq_desc *desc) { }
493 #endif
494 
cond_unmask_eoi_irq(struct irq_desc * desc,struct irq_chip * chip)495 static void cond_unmask_eoi_irq(struct irq_desc *desc, struct irq_chip *chip)
496 {
497 	if (!(desc->istate & IRQS_ONESHOT)) {
498 		chip->irq_eoi(&desc->irq_data);
499 		return;
500 	}
501 	/*
502 	 * We need to unmask in the following cases:
503 	 * - Oneshot irq which did not wake the thread (caused by a
504 	 *   spurious interrupt or a primary handler handling it
505 	 *   completely).
506 	 */
507 	if (!irqd_irq_disabled(&desc->irq_data) &&
508 	    irqd_irq_masked(&desc->irq_data) && !desc->threads_oneshot) {
509 		chip->irq_eoi(&desc->irq_data);
510 		unmask_irq(desc);
511 	} else if (!(chip->flags & IRQCHIP_EOI_THREADED)) {
512 		chip->irq_eoi(&desc->irq_data);
513 	}
514 }
515 
516 /**
517  *	handle_fasteoi_irq - irq handler for transparent controllers
518  *	@desc:	the interrupt description structure for this irq
519  *
520  *	Only a single callback will be issued to the chip: an ->eoi()
521  *	call when the interrupt has been serviced. This enables support
522  *	for modern forms of interrupt handlers, which handle the flow
523  *	details in hardware, transparently.
524  */
handle_fasteoi_irq(struct irq_desc * desc)525 void handle_fasteoi_irq(struct irq_desc *desc)
526 {
527 	struct irq_chip *chip = desc->irq_data.chip;
528 
529 	raw_spin_lock(&desc->lock);
530 
531 	if (!irq_may_run(desc))
532 		goto out;
533 
534 	desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
535 
536 	/*
537 	 * If its disabled or no action available
538 	 * then mask it and get out of here:
539 	 */
540 	if (unlikely(!desc->action || irqd_irq_disabled(&desc->irq_data))) {
541 		desc->istate |= IRQS_PENDING;
542 		mask_irq(desc);
543 		goto out;
544 	}
545 
546 	kstat_incr_irqs_this_cpu(desc);
547 	if (desc->istate & IRQS_ONESHOT)
548 		mask_irq(desc);
549 
550 	preflow_handler(desc);
551 	handle_irq_event(desc);
552 
553 	cond_unmask_eoi_irq(desc, chip);
554 
555 	raw_spin_unlock(&desc->lock);
556 	return;
557 out:
558 	if (!(chip->flags & IRQCHIP_EOI_IF_HANDLED))
559 		chip->irq_eoi(&desc->irq_data);
560 	raw_spin_unlock(&desc->lock);
561 }
562 EXPORT_SYMBOL_GPL(handle_fasteoi_irq);
563 
564 /**
565  *	handle_edge_irq - edge type IRQ handler
566  *	@desc:	the interrupt description structure for this irq
567  *
568  *	Interrupt occures on the falling and/or rising edge of a hardware
569  *	signal. The occurrence is latched into the irq controller hardware
570  *	and must be acked in order to be reenabled. After the ack another
571  *	interrupt can happen on the same source even before the first one
572  *	is handled by the associated event handler. If this happens it
573  *	might be necessary to disable (mask) the interrupt depending on the
574  *	controller hardware. This requires to reenable the interrupt inside
575  *	of the loop which handles the interrupts which have arrived while
576  *	the handler was running. If all pending interrupts are handled, the
577  *	loop is left.
578  */
handle_edge_irq(struct irq_desc * desc)579 void handle_edge_irq(struct irq_desc *desc)
580 {
581 	raw_spin_lock(&desc->lock);
582 
583 	desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
584 
585 	if (!irq_may_run(desc)) {
586 		desc->istate |= IRQS_PENDING;
587 		mask_ack_irq(desc);
588 		goto out_unlock;
589 	}
590 
591 	/*
592 	 * If its disabled or no action available then mask it and get
593 	 * out of here.
594 	 */
595 	if (irqd_irq_disabled(&desc->irq_data) || !desc->action) {
596 		desc->istate |= IRQS_PENDING;
597 		mask_ack_irq(desc);
598 		goto out_unlock;
599 	}
600 
601 	kstat_incr_irqs_this_cpu(desc);
602 
603 	/* Start handling the irq */
604 	desc->irq_data.chip->irq_ack(&desc->irq_data);
605 
606 	do {
607 		if (unlikely(!desc->action)) {
608 			mask_irq(desc);
609 			goto out_unlock;
610 		}
611 
612 		/*
613 		 * When another irq arrived while we were handling
614 		 * one, we could have masked the irq.
615 		 * Renable it, if it was not disabled in meantime.
616 		 */
617 		if (unlikely(desc->istate & IRQS_PENDING)) {
618 			if (!irqd_irq_disabled(&desc->irq_data) &&
619 			    irqd_irq_masked(&desc->irq_data))
620 				unmask_irq(desc);
621 		}
622 
623 		handle_irq_event(desc);
624 
625 	} while ((desc->istate & IRQS_PENDING) &&
626 		 !irqd_irq_disabled(&desc->irq_data));
627 
628 out_unlock:
629 	raw_spin_unlock(&desc->lock);
630 }
631 EXPORT_SYMBOL(handle_edge_irq);
632 
633 #ifdef CONFIG_IRQ_EDGE_EOI_HANDLER
634 /**
635  *	handle_edge_eoi_irq - edge eoi type IRQ handler
636  *	@desc:	the interrupt description structure for this irq
637  *
638  * Similar as the above handle_edge_irq, but using eoi and w/o the
639  * mask/unmask logic.
640  */
handle_edge_eoi_irq(struct irq_desc * desc)641 void handle_edge_eoi_irq(struct irq_desc *desc)
642 {
643 	struct irq_chip *chip = irq_desc_get_chip(desc);
644 
645 	raw_spin_lock(&desc->lock);
646 
647 	desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
648 
649 	if (!irq_may_run(desc)) {
650 		desc->istate |= IRQS_PENDING;
651 		goto out_eoi;
652 	}
653 
654 	/*
655 	 * If its disabled or no action available then mask it and get
656 	 * out of here.
657 	 */
658 	if (irqd_irq_disabled(&desc->irq_data) || !desc->action) {
659 		desc->istate |= IRQS_PENDING;
660 		goto out_eoi;
661 	}
662 
663 	kstat_incr_irqs_this_cpu(desc);
664 
665 	do {
666 		if (unlikely(!desc->action))
667 			goto out_eoi;
668 
669 		handle_irq_event(desc);
670 
671 	} while ((desc->istate & IRQS_PENDING) &&
672 		 !irqd_irq_disabled(&desc->irq_data));
673 
674 out_eoi:
675 	chip->irq_eoi(&desc->irq_data);
676 	raw_spin_unlock(&desc->lock);
677 }
678 #endif
679 
680 /**
681  *	handle_percpu_irq - Per CPU local irq handler
682  *	@desc:	the interrupt description structure for this irq
683  *
684  *	Per CPU interrupts on SMP machines without locking requirements
685  */
handle_percpu_irq(struct irq_desc * desc)686 void handle_percpu_irq(struct irq_desc *desc)
687 {
688 	struct irq_chip *chip = irq_desc_get_chip(desc);
689 
690 	kstat_incr_irqs_this_cpu(desc);
691 
692 	if (chip->irq_ack)
693 		chip->irq_ack(&desc->irq_data);
694 
695 	handle_irq_event_percpu(desc);
696 
697 	if (chip->irq_eoi)
698 		chip->irq_eoi(&desc->irq_data);
699 }
700 
701 /**
702  * handle_percpu_devid_irq - Per CPU local irq handler with per cpu dev ids
703  * @desc:	the interrupt description structure for this irq
704  *
705  * Per CPU interrupts on SMP machines without locking requirements. Same as
706  * handle_percpu_irq() above but with the following extras:
707  *
708  * action->percpu_dev_id is a pointer to percpu variables which
709  * contain the real device id for the cpu on which this handler is
710  * called
711  */
handle_percpu_devid_irq(struct irq_desc * desc)712 void handle_percpu_devid_irq(struct irq_desc *desc)
713 {
714 	struct irq_chip *chip = irq_desc_get_chip(desc);
715 	struct irqaction *action = desc->action;
716 	void *dev_id = raw_cpu_ptr(action->percpu_dev_id);
717 	unsigned int irq = irq_desc_get_irq(desc);
718 	irqreturn_t res;
719 
720 	kstat_incr_irqs_this_cpu(desc);
721 
722 	if (chip->irq_ack)
723 		chip->irq_ack(&desc->irq_data);
724 
725 	trace_irq_handler_entry(irq, action);
726 	res = action->handler(irq, dev_id);
727 	trace_irq_handler_exit(irq, action, res);
728 
729 	if (chip->irq_eoi)
730 		chip->irq_eoi(&desc->irq_data);
731 }
732 
733 void
__irq_do_set_handler(struct irq_desc * desc,irq_flow_handler_t handle,int is_chained,const char * name)734 __irq_do_set_handler(struct irq_desc *desc, irq_flow_handler_t handle,
735 		     int is_chained, const char *name)
736 {
737 	if (!handle) {
738 		handle = handle_bad_irq;
739 	} else {
740 		struct irq_data *irq_data = &desc->irq_data;
741 #ifdef CONFIG_IRQ_DOMAIN_HIERARCHY
742 		/*
743 		 * With hierarchical domains we might run into a
744 		 * situation where the outermost chip is not yet set
745 		 * up, but the inner chips are there.  Instead of
746 		 * bailing we install the handler, but obviously we
747 		 * cannot enable/startup the interrupt at this point.
748 		 */
749 		while (irq_data) {
750 			if (irq_data->chip != &no_irq_chip)
751 				break;
752 			/*
753 			 * Bail out if the outer chip is not set up
754 			 * and the interrrupt supposed to be started
755 			 * right away.
756 			 */
757 			if (WARN_ON(is_chained))
758 				return;
759 			/* Try the parent */
760 			irq_data = irq_data->parent_data;
761 		}
762 #endif
763 		if (WARN_ON(!irq_data || irq_data->chip == &no_irq_chip))
764 			return;
765 	}
766 
767 	/* Uninstall? */
768 	if (handle == handle_bad_irq) {
769 		if (desc->irq_data.chip != &no_irq_chip)
770 			mask_ack_irq(desc);
771 		irq_state_set_disabled(desc);
772 		if (is_chained)
773 			desc->action = NULL;
774 		desc->depth = 1;
775 	}
776 	desc->handle_irq = handle;
777 	desc->name = name;
778 
779 	if (handle != handle_bad_irq && is_chained) {
780 		irq_settings_set_noprobe(desc);
781 		irq_settings_set_norequest(desc);
782 		irq_settings_set_nothread(desc);
783 		desc->action = &chained_action;
784 		irq_startup(desc, true);
785 	}
786 }
787 
788 void
__irq_set_handler(unsigned int irq,irq_flow_handler_t handle,int is_chained,const char * name)789 __irq_set_handler(unsigned int irq, irq_flow_handler_t handle, int is_chained,
790 		  const char *name)
791 {
792 	unsigned long flags;
793 	struct irq_desc *desc = irq_get_desc_buslock(irq, &flags, 0);
794 
795 	if (!desc)
796 		return;
797 
798 	__irq_do_set_handler(desc, handle, is_chained, name);
799 	irq_put_desc_busunlock(desc, flags);
800 }
801 EXPORT_SYMBOL_GPL(__irq_set_handler);
802 
803 void
irq_set_chained_handler_and_data(unsigned int irq,irq_flow_handler_t handle,void * data)804 irq_set_chained_handler_and_data(unsigned int irq, irq_flow_handler_t handle,
805 				 void *data)
806 {
807 	unsigned long flags;
808 	struct irq_desc *desc = irq_get_desc_buslock(irq, &flags, 0);
809 
810 	if (!desc)
811 		return;
812 
813 	desc->irq_common_data.handler_data = data;
814 	__irq_do_set_handler(desc, handle, 1, NULL);
815 
816 	irq_put_desc_busunlock(desc, flags);
817 }
818 EXPORT_SYMBOL_GPL(irq_set_chained_handler_and_data);
819 
820 void
irq_set_chip_and_handler_name(unsigned int irq,struct irq_chip * chip,irq_flow_handler_t handle,const char * name)821 irq_set_chip_and_handler_name(unsigned int irq, struct irq_chip *chip,
822 			      irq_flow_handler_t handle, const char *name)
823 {
824 	irq_set_chip(irq, chip);
825 	__irq_set_handler(irq, handle, 0, name);
826 }
827 EXPORT_SYMBOL_GPL(irq_set_chip_and_handler_name);
828 
irq_modify_status(unsigned int irq,unsigned long clr,unsigned long set)829 void irq_modify_status(unsigned int irq, unsigned long clr, unsigned long set)
830 {
831 	unsigned long flags;
832 	struct irq_desc *desc = irq_get_desc_lock(irq, &flags, 0);
833 
834 	if (!desc)
835 		return;
836 	irq_settings_clr_and_set(desc, clr, set);
837 
838 	irqd_clear(&desc->irq_data, IRQD_NO_BALANCING | IRQD_PER_CPU |
839 		   IRQD_TRIGGER_MASK | IRQD_LEVEL | IRQD_MOVE_PCNTXT);
840 	if (irq_settings_has_no_balance_set(desc))
841 		irqd_set(&desc->irq_data, IRQD_NO_BALANCING);
842 	if (irq_settings_is_per_cpu(desc))
843 		irqd_set(&desc->irq_data, IRQD_PER_CPU);
844 	if (irq_settings_can_move_pcntxt(desc))
845 		irqd_set(&desc->irq_data, IRQD_MOVE_PCNTXT);
846 	if (irq_settings_is_level(desc))
847 		irqd_set(&desc->irq_data, IRQD_LEVEL);
848 
849 	irqd_set(&desc->irq_data, irq_settings_get_trigger_mask(desc));
850 
851 	irq_put_desc_unlock(desc, flags);
852 }
853 EXPORT_SYMBOL_GPL(irq_modify_status);
854 
855 /**
856  *	irq_cpu_online - Invoke all irq_cpu_online functions.
857  *
858  *	Iterate through all irqs and invoke the chip.irq_cpu_online()
859  *	for each.
860  */
irq_cpu_online(void)861 void irq_cpu_online(void)
862 {
863 	struct irq_desc *desc;
864 	struct irq_chip *chip;
865 	unsigned long flags;
866 	unsigned int irq;
867 
868 	for_each_active_irq(irq) {
869 		desc = irq_to_desc(irq);
870 		if (!desc)
871 			continue;
872 
873 		raw_spin_lock_irqsave(&desc->lock, flags);
874 
875 		chip = irq_data_get_irq_chip(&desc->irq_data);
876 		if (chip && chip->irq_cpu_online &&
877 		    (!(chip->flags & IRQCHIP_ONOFFLINE_ENABLED) ||
878 		     !irqd_irq_disabled(&desc->irq_data)))
879 			chip->irq_cpu_online(&desc->irq_data);
880 
881 		raw_spin_unlock_irqrestore(&desc->lock, flags);
882 	}
883 }
884 
885 /**
886  *	irq_cpu_offline - Invoke all irq_cpu_offline functions.
887  *
888  *	Iterate through all irqs and invoke the chip.irq_cpu_offline()
889  *	for each.
890  */
irq_cpu_offline(void)891 void irq_cpu_offline(void)
892 {
893 	struct irq_desc *desc;
894 	struct irq_chip *chip;
895 	unsigned long flags;
896 	unsigned int irq;
897 
898 	for_each_active_irq(irq) {
899 		desc = irq_to_desc(irq);
900 		if (!desc)
901 			continue;
902 
903 		raw_spin_lock_irqsave(&desc->lock, flags);
904 
905 		chip = irq_data_get_irq_chip(&desc->irq_data);
906 		if (chip && chip->irq_cpu_offline &&
907 		    (!(chip->flags & IRQCHIP_ONOFFLINE_ENABLED) ||
908 		     !irqd_irq_disabled(&desc->irq_data)))
909 			chip->irq_cpu_offline(&desc->irq_data);
910 
911 		raw_spin_unlock_irqrestore(&desc->lock, flags);
912 	}
913 }
914 
915 #ifdef	CONFIG_IRQ_DOMAIN_HIERARCHY
916 /**
917  * irq_chip_enable_parent - Enable the parent interrupt (defaults to unmask if
918  * NULL)
919  * @data:	Pointer to interrupt specific data
920  */
irq_chip_enable_parent(struct irq_data * data)921 void irq_chip_enable_parent(struct irq_data *data)
922 {
923 	data = data->parent_data;
924 	if (data->chip->irq_enable)
925 		data->chip->irq_enable(data);
926 	else
927 		data->chip->irq_unmask(data);
928 }
929 
930 /**
931  * irq_chip_disable_parent - Disable the parent interrupt (defaults to mask if
932  * NULL)
933  * @data:	Pointer to interrupt specific data
934  */
irq_chip_disable_parent(struct irq_data * data)935 void irq_chip_disable_parent(struct irq_data *data)
936 {
937 	data = data->parent_data;
938 	if (data->chip->irq_disable)
939 		data->chip->irq_disable(data);
940 	else
941 		data->chip->irq_mask(data);
942 }
943 
944 /**
945  * irq_chip_ack_parent - Acknowledge the parent interrupt
946  * @data:	Pointer to interrupt specific data
947  */
irq_chip_ack_parent(struct irq_data * data)948 void irq_chip_ack_parent(struct irq_data *data)
949 {
950 	data = data->parent_data;
951 	data->chip->irq_ack(data);
952 }
953 
954 /**
955  * irq_chip_mask_parent - Mask the parent interrupt
956  * @data:	Pointer to interrupt specific data
957  */
irq_chip_mask_parent(struct irq_data * data)958 void irq_chip_mask_parent(struct irq_data *data)
959 {
960 	data = data->parent_data;
961 	data->chip->irq_mask(data);
962 }
963 
964 /**
965  * irq_chip_unmask_parent - Unmask the parent interrupt
966  * @data:	Pointer to interrupt specific data
967  */
irq_chip_unmask_parent(struct irq_data * data)968 void irq_chip_unmask_parent(struct irq_data *data)
969 {
970 	data = data->parent_data;
971 	data->chip->irq_unmask(data);
972 }
973 
974 /**
975  * irq_chip_eoi_parent - Invoke EOI on the parent interrupt
976  * @data:	Pointer to interrupt specific data
977  */
irq_chip_eoi_parent(struct irq_data * data)978 void irq_chip_eoi_parent(struct irq_data *data)
979 {
980 	data = data->parent_data;
981 	data->chip->irq_eoi(data);
982 }
983 
984 /**
985  * irq_chip_set_affinity_parent - Set affinity on the parent interrupt
986  * @data:	Pointer to interrupt specific data
987  * @dest:	The affinity mask to set
988  * @force:	Flag to enforce setting (disable online checks)
989  *
990  * Conditinal, as the underlying parent chip might not implement it.
991  */
irq_chip_set_affinity_parent(struct irq_data * data,const struct cpumask * dest,bool force)992 int irq_chip_set_affinity_parent(struct irq_data *data,
993 				 const struct cpumask *dest, bool force)
994 {
995 	data = data->parent_data;
996 	if (data->chip->irq_set_affinity)
997 		return data->chip->irq_set_affinity(data, dest, force);
998 
999 	return -ENOSYS;
1000 }
1001 
1002 /**
1003  * irq_chip_set_type_parent - Set IRQ type on the parent interrupt
1004  * @data:	Pointer to interrupt specific data
1005  * @type:	IRQ_TYPE_{LEVEL,EDGE}_* value - see include/linux/irq.h
1006  *
1007  * Conditional, as the underlying parent chip might not implement it.
1008  */
irq_chip_set_type_parent(struct irq_data * data,unsigned int type)1009 int irq_chip_set_type_parent(struct irq_data *data, unsigned int type)
1010 {
1011 	data = data->parent_data;
1012 
1013 	if (data->chip->irq_set_type)
1014 		return data->chip->irq_set_type(data, type);
1015 
1016 	return -ENOSYS;
1017 }
1018 
1019 /**
1020  * irq_chip_retrigger_hierarchy - Retrigger an interrupt in hardware
1021  * @data:	Pointer to interrupt specific data
1022  *
1023  * Iterate through the domain hierarchy of the interrupt and check
1024  * whether a hw retrigger function exists. If yes, invoke it.
1025  */
irq_chip_retrigger_hierarchy(struct irq_data * data)1026 int irq_chip_retrigger_hierarchy(struct irq_data *data)
1027 {
1028 	for (data = data->parent_data; data; data = data->parent_data)
1029 		if (data->chip && data->chip->irq_retrigger)
1030 			return data->chip->irq_retrigger(data);
1031 
1032 	return 0;
1033 }
1034 
1035 /**
1036  * irq_chip_set_vcpu_affinity_parent - Set vcpu affinity on the parent interrupt
1037  * @data:	Pointer to interrupt specific data
1038  * @vcpu_info:	The vcpu affinity information
1039  */
irq_chip_set_vcpu_affinity_parent(struct irq_data * data,void * vcpu_info)1040 int irq_chip_set_vcpu_affinity_parent(struct irq_data *data, void *vcpu_info)
1041 {
1042 	data = data->parent_data;
1043 	if (data->chip->irq_set_vcpu_affinity)
1044 		return data->chip->irq_set_vcpu_affinity(data, vcpu_info);
1045 
1046 	return -ENOSYS;
1047 }
1048 
1049 /**
1050  * irq_chip_set_wake_parent - Set/reset wake-up on the parent interrupt
1051  * @data:	Pointer to interrupt specific data
1052  * @on:		Whether to set or reset the wake-up capability of this irq
1053  *
1054  * Conditional, as the underlying parent chip might not implement it.
1055  */
irq_chip_set_wake_parent(struct irq_data * data,unsigned int on)1056 int irq_chip_set_wake_parent(struct irq_data *data, unsigned int on)
1057 {
1058 	data = data->parent_data;
1059 
1060 	if (data->chip->flags & IRQCHIP_SKIP_SET_WAKE)
1061 		return 0;
1062 
1063 	if (data->chip->irq_set_wake)
1064 		return data->chip->irq_set_wake(data, on);
1065 
1066 	return -ENOSYS;
1067 }
1068 #endif
1069 
1070 /**
1071  * irq_chip_compose_msi_msg - Componse msi message for a irq chip
1072  * @data:	Pointer to interrupt specific data
1073  * @msg:	Pointer to the MSI message
1074  *
1075  * For hierarchical domains we find the first chip in the hierarchy
1076  * which implements the irq_compose_msi_msg callback. For non
1077  * hierarchical we use the top level chip.
1078  */
irq_chip_compose_msi_msg(struct irq_data * data,struct msi_msg * msg)1079 int irq_chip_compose_msi_msg(struct irq_data *data, struct msi_msg *msg)
1080 {
1081 	struct irq_data *pos = NULL;
1082 
1083 #ifdef	CONFIG_IRQ_DOMAIN_HIERARCHY
1084 	for (; data; data = data->parent_data)
1085 #endif
1086 		if (data->chip && data->chip->irq_compose_msi_msg)
1087 			pos = data;
1088 	if (!pos)
1089 		return -ENOSYS;
1090 
1091 	pos->chip->irq_compose_msi_msg(pos, msg);
1092 
1093 	return 0;
1094 }
1095