• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (C) 1992, 1998-2006 Linus Torvalds, Ingo Molnar
4  * Copyright (C) 2005-2006, Thomas Gleixner, Russell King
5  *
6  * This file contains the core interrupt handling code, for irq-chip based
7  * architectures. Detailed information is available in
8  * Documentation/core-api/genericirq.rst
9  */
10 
11 #include <linux/irq.h>
12 #include <linux/msi.h>
13 #include <linux/module.h>
14 #include <linux/interrupt.h>
15 #include <linux/kernel_stat.h>
16 #include <linux/irqdomain.h>
17 
18 #include <trace/events/irq.h>
19 
20 #include "internals.h"
21 
bad_chained_irq(int irq,void * dev_id)22 static irqreturn_t bad_chained_irq(int irq, void *dev_id)
23 {
24 	WARN_ONCE(1, "Chained irq %d should not call an action\n", irq);
25 	return IRQ_NONE;
26 }
27 
28 /*
29  * Chained handlers should never call action on their IRQ. This default
30  * action will emit warning if such thing happens.
31  */
32 struct irqaction chained_action = {
33 	.handler = bad_chained_irq,
34 };
35 
36 /**
37  *	irq_set_chip - set the irq chip for an irq
38  *	@irq:	irq number
39  *	@chip:	pointer to irq chip description structure
40  */
irq_set_chip(unsigned int irq,struct irq_chip * chip)41 int irq_set_chip(unsigned int irq, struct irq_chip *chip)
42 {
43 	unsigned long flags;
44 	struct irq_desc *desc = irq_get_desc_lock(irq, &flags, 0);
45 
46 	if (!desc)
47 		return -EINVAL;
48 
49 	if (!chip)
50 		chip = &no_irq_chip;
51 
52 	desc->irq_data.chip = chip;
53 	irq_put_desc_unlock(desc, flags);
54 	/*
55 	 * For !CONFIG_SPARSE_IRQ make the irq show up in
56 	 * allocated_irqs.
57 	 */
58 	irq_mark_irq(irq);
59 	return 0;
60 }
61 EXPORT_SYMBOL(irq_set_chip);
62 
63 /**
64  *	irq_set_type - set the irq trigger type for an irq
65  *	@irq:	irq number
66  *	@type:	IRQ_TYPE_{LEVEL,EDGE}_* value - see include/linux/irq.h
67  */
irq_set_irq_type(unsigned int irq,unsigned int type)68 int irq_set_irq_type(unsigned int irq, unsigned int type)
69 {
70 	unsigned long flags;
71 	struct irq_desc *desc = irq_get_desc_buslock(irq, &flags, IRQ_GET_DESC_CHECK_GLOBAL);
72 	int ret = 0;
73 
74 	if (!desc)
75 		return -EINVAL;
76 
77 	ret = __irq_set_trigger(desc, type);
78 	irq_put_desc_busunlock(desc, flags);
79 	return ret;
80 }
81 EXPORT_SYMBOL(irq_set_irq_type);
82 
83 /**
84  *	irq_set_handler_data - set irq handler data for an irq
85  *	@irq:	Interrupt number
86  *	@data:	Pointer to interrupt specific data
87  *
88  *	Set the hardware irq controller data for an irq
89  */
irq_set_handler_data(unsigned int irq,void * data)90 int irq_set_handler_data(unsigned int irq, void *data)
91 {
92 	unsigned long flags;
93 	struct irq_desc *desc = irq_get_desc_lock(irq, &flags, 0);
94 
95 	if (!desc)
96 		return -EINVAL;
97 	desc->irq_common_data.handler_data = data;
98 	irq_put_desc_unlock(desc, flags);
99 	return 0;
100 }
101 EXPORT_SYMBOL(irq_set_handler_data);
102 
103 /**
104  *	irq_set_msi_desc_off - set MSI descriptor data for an irq at offset
105  *	@irq_base:	Interrupt number base
106  *	@irq_offset:	Interrupt number offset
107  *	@entry:		Pointer to MSI descriptor data
108  *
109  *	Set the MSI descriptor entry for an irq at offset
110  */
irq_set_msi_desc_off(unsigned int irq_base,unsigned int irq_offset,struct msi_desc * entry)111 int irq_set_msi_desc_off(unsigned int irq_base, unsigned int irq_offset,
112 			 struct msi_desc *entry)
113 {
114 	unsigned long flags;
115 	struct irq_desc *desc = irq_get_desc_lock(irq_base + irq_offset, &flags, IRQ_GET_DESC_CHECK_GLOBAL);
116 
117 	if (!desc)
118 		return -EINVAL;
119 	desc->irq_common_data.msi_desc = entry;
120 	if (entry && !irq_offset)
121 		entry->irq = irq_base;
122 	irq_put_desc_unlock(desc, flags);
123 	return 0;
124 }
125 
126 /**
127  *	irq_set_msi_desc - set MSI descriptor data for an irq
128  *	@irq:	Interrupt number
129  *	@entry:	Pointer to MSI descriptor data
130  *
131  *	Set the MSI descriptor entry for an irq
132  */
irq_set_msi_desc(unsigned int irq,struct msi_desc * entry)133 int irq_set_msi_desc(unsigned int irq, struct msi_desc *entry)
134 {
135 	return irq_set_msi_desc_off(irq, 0, entry);
136 }
137 
138 /**
139  *	irq_set_chip_data - set irq chip data for an irq
140  *	@irq:	Interrupt number
141  *	@data:	Pointer to chip specific data
142  *
143  *	Set the hardware irq chip data for an irq
144  */
irq_set_chip_data(unsigned int irq,void * data)145 int irq_set_chip_data(unsigned int irq, void *data)
146 {
147 	unsigned long flags;
148 	struct irq_desc *desc = irq_get_desc_lock(irq, &flags, 0);
149 
150 	if (!desc)
151 		return -EINVAL;
152 	desc->irq_data.chip_data = data;
153 	irq_put_desc_unlock(desc, flags);
154 	return 0;
155 }
156 EXPORT_SYMBOL(irq_set_chip_data);
157 
irq_get_irq_data(unsigned int irq)158 struct irq_data *irq_get_irq_data(unsigned int irq)
159 {
160 	struct irq_desc *desc = irq_to_desc(irq);
161 
162 	return desc ? &desc->irq_data : NULL;
163 }
164 EXPORT_SYMBOL_GPL(irq_get_irq_data);
165 
irq_state_clr_disabled(struct irq_desc * desc)166 static void irq_state_clr_disabled(struct irq_desc *desc)
167 {
168 	irqd_clear(&desc->irq_data, IRQD_IRQ_DISABLED);
169 }
170 
irq_state_clr_masked(struct irq_desc * desc)171 static void irq_state_clr_masked(struct irq_desc *desc)
172 {
173 	irqd_clear(&desc->irq_data, IRQD_IRQ_MASKED);
174 }
175 
irq_state_clr_started(struct irq_desc * desc)176 static void irq_state_clr_started(struct irq_desc *desc)
177 {
178 	irqd_clear(&desc->irq_data, IRQD_IRQ_STARTED);
179 }
180 
irq_state_set_started(struct irq_desc * desc)181 static void irq_state_set_started(struct irq_desc *desc)
182 {
183 	irqd_set(&desc->irq_data, IRQD_IRQ_STARTED);
184 }
185 
186 enum {
187 	IRQ_STARTUP_NORMAL,
188 	IRQ_STARTUP_MANAGED,
189 	IRQ_STARTUP_ABORT,
190 };
191 
192 #ifdef CONFIG_SMP
193 static int
__irq_startup_managed(struct irq_desc * desc,struct cpumask * aff,bool force)194 __irq_startup_managed(struct irq_desc *desc, struct cpumask *aff, bool force)
195 {
196 	struct irq_data *d = irq_desc_get_irq_data(desc);
197 
198 	if (!irqd_affinity_is_managed(d))
199 		return IRQ_STARTUP_NORMAL;
200 
201 	irqd_clr_managed_shutdown(d);
202 
203 	if (cpumask_any_and(aff, cpu_online_mask) >= nr_cpu_ids) {
204 		/*
205 		 * Catch code which fiddles with enable_irq() on a managed
206 		 * and potentially shutdown IRQ. Chained interrupt
207 		 * installment or irq auto probing should not happen on
208 		 * managed irqs either.
209 		 */
210 		if (WARN_ON_ONCE(force))
211 			return IRQ_STARTUP_ABORT;
212 		/*
213 		 * The interrupt was requested, but there is no online CPU
214 		 * in it's affinity mask. Put it into managed shutdown
215 		 * state and let the cpu hotplug mechanism start it up once
216 		 * a CPU in the mask becomes available.
217 		 */
218 		return IRQ_STARTUP_ABORT;
219 	}
220 	/*
221 	 * Managed interrupts have reserved resources, so this should not
222 	 * happen.
223 	 */
224 	if (WARN_ON(irq_domain_activate_irq(d, false)))
225 		return IRQ_STARTUP_ABORT;
226 	return IRQ_STARTUP_MANAGED;
227 }
228 #else
229 static __always_inline int
__irq_startup_managed(struct irq_desc * desc,struct cpumask * aff,bool force)230 __irq_startup_managed(struct irq_desc *desc, struct cpumask *aff, bool force)
231 {
232 	return IRQ_STARTUP_NORMAL;
233 }
234 #endif
235 
__irq_startup(struct irq_desc * desc)236 static int __irq_startup(struct irq_desc *desc)
237 {
238 	struct irq_data *d = irq_desc_get_irq_data(desc);
239 	int ret = 0;
240 
241 	/* Warn if this interrupt is not activated but try nevertheless */
242 	WARN_ON_ONCE(!irqd_is_activated(d));
243 
244 	if (d->chip->irq_startup) {
245 		ret = d->chip->irq_startup(d);
246 		irq_state_clr_disabled(desc);
247 		irq_state_clr_masked(desc);
248 	} else {
249 		irq_enable(desc);
250 	}
251 	irq_state_set_started(desc);
252 	return ret;
253 }
254 
irq_startup(struct irq_desc * desc,bool resend,bool force)255 int irq_startup(struct irq_desc *desc, bool resend, bool force)
256 {
257 	struct irq_data *d = irq_desc_get_irq_data(desc);
258 	struct cpumask *aff = irq_data_get_affinity_mask(d);
259 	int ret = 0;
260 
261 	desc->depth = 0;
262 
263 	if (irqd_is_started(d)) {
264 		irq_enable(desc);
265 	} else {
266 		switch (__irq_startup_managed(desc, aff, force)) {
267 		case IRQ_STARTUP_NORMAL:
268 			if (d->chip->flags & IRQCHIP_AFFINITY_PRE_STARTUP)
269 				irq_setup_affinity(desc);
270 			ret = __irq_startup(desc);
271 			if (!(d->chip->flags & IRQCHIP_AFFINITY_PRE_STARTUP))
272 				irq_setup_affinity(desc);
273 			break;
274 		case IRQ_STARTUP_MANAGED:
275 			irq_do_set_affinity(d, aff, false);
276 			ret = __irq_startup(desc);
277 			break;
278 		case IRQ_STARTUP_ABORT:
279 			irqd_set_managed_shutdown(d);
280 			return 0;
281 		}
282 	}
283 	if (resend)
284 		check_irq_resend(desc, false);
285 
286 	return ret;
287 }
288 
irq_activate(struct irq_desc * desc)289 int irq_activate(struct irq_desc *desc)
290 {
291 	struct irq_data *d = irq_desc_get_irq_data(desc);
292 
293 	if (!irqd_affinity_is_managed(d))
294 		return irq_domain_activate_irq(d, false);
295 	return 0;
296 }
297 
irq_activate_and_startup(struct irq_desc * desc,bool resend)298 int irq_activate_and_startup(struct irq_desc *desc, bool resend)
299 {
300 	if (WARN_ON(irq_activate(desc)))
301 		return 0;
302 	return irq_startup(desc, resend, IRQ_START_FORCE);
303 }
304 
305 static void __irq_disable(struct irq_desc *desc, bool mask);
306 
irq_shutdown(struct irq_desc * desc)307 void irq_shutdown(struct irq_desc *desc)
308 {
309 	if (irqd_is_started(&desc->irq_data)) {
310 		desc->depth = 1;
311 		if (desc->irq_data.chip->irq_shutdown) {
312 			desc->irq_data.chip->irq_shutdown(&desc->irq_data);
313 			irq_state_set_disabled(desc);
314 			irq_state_set_masked(desc);
315 		} else {
316 			__irq_disable(desc, true);
317 		}
318 		irq_state_clr_started(desc);
319 	}
320 }
321 
322 
irq_shutdown_and_deactivate(struct irq_desc * desc)323 void irq_shutdown_and_deactivate(struct irq_desc *desc)
324 {
325 	irq_shutdown(desc);
326 	/*
327 	 * This must be called even if the interrupt was never started up,
328 	 * because the activation can happen before the interrupt is
329 	 * available for request/startup. It has it's own state tracking so
330 	 * it's safe to call it unconditionally.
331 	 */
332 	irq_domain_deactivate_irq(&desc->irq_data);
333 }
334 
irq_enable(struct irq_desc * desc)335 void irq_enable(struct irq_desc *desc)
336 {
337 	if (!irqd_irq_disabled(&desc->irq_data)) {
338 		unmask_irq(desc);
339 	} else {
340 		irq_state_clr_disabled(desc);
341 		if (desc->irq_data.chip->irq_enable) {
342 			desc->irq_data.chip->irq_enable(&desc->irq_data);
343 			irq_state_clr_masked(desc);
344 		} else {
345 			unmask_irq(desc);
346 		}
347 	}
348 }
349 
__irq_disable(struct irq_desc * desc,bool mask)350 static void __irq_disable(struct irq_desc *desc, bool mask)
351 {
352 	if (irqd_irq_disabled(&desc->irq_data)) {
353 		if (mask)
354 			mask_irq(desc);
355 	} else {
356 		irq_state_set_disabled(desc);
357 		if (desc->irq_data.chip->irq_disable) {
358 			desc->irq_data.chip->irq_disable(&desc->irq_data);
359 			irq_state_set_masked(desc);
360 		} else if (mask) {
361 			mask_irq(desc);
362 		}
363 	}
364 }
365 
366 /**
367  * irq_disable - Mark interrupt disabled
368  * @desc:	irq descriptor which should be disabled
369  *
370  * If the chip does not implement the irq_disable callback, we
371  * use a lazy disable approach. That means we mark the interrupt
372  * disabled, but leave the hardware unmasked. That's an
373  * optimization because we avoid the hardware access for the
374  * common case where no interrupt happens after we marked it
375  * disabled. If an interrupt happens, then the interrupt flow
376  * handler masks the line at the hardware level and marks it
377  * pending.
378  *
379  * If the interrupt chip does not implement the irq_disable callback,
380  * a driver can disable the lazy approach for a particular irq line by
381  * calling 'irq_set_status_flags(irq, IRQ_DISABLE_UNLAZY)'. This can
382  * be used for devices which cannot disable the interrupt at the
383  * device level under certain circumstances and have to use
384  * disable_irq[_nosync] instead.
385  */
irq_disable(struct irq_desc * desc)386 void irq_disable(struct irq_desc *desc)
387 {
388 	__irq_disable(desc, irq_settings_disable_unlazy(desc));
389 }
390 
irq_percpu_enable(struct irq_desc * desc,unsigned int cpu)391 void irq_percpu_enable(struct irq_desc *desc, unsigned int cpu)
392 {
393 	if (desc->irq_data.chip->irq_enable)
394 		desc->irq_data.chip->irq_enable(&desc->irq_data);
395 	else
396 		desc->irq_data.chip->irq_unmask(&desc->irq_data);
397 	cpumask_set_cpu(cpu, desc->percpu_enabled);
398 }
399 
irq_percpu_disable(struct irq_desc * desc,unsigned int cpu)400 void irq_percpu_disable(struct irq_desc *desc, unsigned int cpu)
401 {
402 	if (desc->irq_data.chip->irq_disable)
403 		desc->irq_data.chip->irq_disable(&desc->irq_data);
404 	else
405 		desc->irq_data.chip->irq_mask(&desc->irq_data);
406 	cpumask_clear_cpu(cpu, desc->percpu_enabled);
407 }
408 
mask_ack_irq(struct irq_desc * desc)409 static inline void mask_ack_irq(struct irq_desc *desc)
410 {
411 	if (desc->irq_data.chip->irq_mask_ack) {
412 		desc->irq_data.chip->irq_mask_ack(&desc->irq_data);
413 		irq_state_set_masked(desc);
414 	} else {
415 		mask_irq(desc);
416 		if (desc->irq_data.chip->irq_ack)
417 			desc->irq_data.chip->irq_ack(&desc->irq_data);
418 	}
419 }
420 
mask_irq(struct irq_desc * desc)421 void mask_irq(struct irq_desc *desc)
422 {
423 	if (irqd_irq_masked(&desc->irq_data))
424 		return;
425 
426 	if (desc->irq_data.chip->irq_mask) {
427 		desc->irq_data.chip->irq_mask(&desc->irq_data);
428 		irq_state_set_masked(desc);
429 	}
430 }
431 
unmask_irq(struct irq_desc * desc)432 void unmask_irq(struct irq_desc *desc)
433 {
434 	if (!irqd_irq_masked(&desc->irq_data))
435 		return;
436 
437 	if (desc->irq_data.chip->irq_unmask) {
438 		desc->irq_data.chip->irq_unmask(&desc->irq_data);
439 		irq_state_clr_masked(desc);
440 	}
441 }
442 
unmask_threaded_irq(struct irq_desc * desc)443 void unmask_threaded_irq(struct irq_desc *desc)
444 {
445 	struct irq_chip *chip = desc->irq_data.chip;
446 
447 	if (chip->flags & IRQCHIP_EOI_THREADED)
448 		chip->irq_eoi(&desc->irq_data);
449 
450 	unmask_irq(desc);
451 }
452 
453 /*
454  *	handle_nested_irq - Handle a nested irq from a irq thread
455  *	@irq:	the interrupt number
456  *
457  *	Handle interrupts which are nested into a threaded interrupt
458  *	handler. The handler function is called inside the calling
459  *	threads context.
460  */
handle_nested_irq(unsigned int irq)461 void handle_nested_irq(unsigned int irq)
462 {
463 	struct irq_desc *desc = irq_to_desc(irq);
464 	struct irqaction *action;
465 	irqreturn_t action_ret;
466 
467 	might_sleep();
468 
469 	raw_spin_lock_irq(&desc->lock);
470 
471 	desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
472 
473 	action = desc->action;
474 	if (unlikely(!action || irqd_irq_disabled(&desc->irq_data))) {
475 		desc->istate |= IRQS_PENDING;
476 		goto out_unlock;
477 	}
478 
479 	kstat_incr_irqs_this_cpu(desc);
480 	irqd_set(&desc->irq_data, IRQD_IRQ_INPROGRESS);
481 	raw_spin_unlock_irq(&desc->lock);
482 
483 	action_ret = IRQ_NONE;
484 	for_each_action_of_desc(desc, action)
485 		action_ret |= action->thread_fn(action->irq, action->dev_id);
486 
487 	if (!noirqdebug)
488 		note_interrupt(desc, action_ret);
489 
490 	raw_spin_lock_irq(&desc->lock);
491 	irqd_clear(&desc->irq_data, IRQD_IRQ_INPROGRESS);
492 
493 out_unlock:
494 	raw_spin_unlock_irq(&desc->lock);
495 }
496 EXPORT_SYMBOL_GPL(handle_nested_irq);
497 
irq_check_poll(struct irq_desc * desc)498 static bool irq_check_poll(struct irq_desc *desc)
499 {
500 	if (!(desc->istate & IRQS_POLL_INPROGRESS))
501 		return false;
502 	return irq_wait_for_poll(desc);
503 }
504 
irq_may_run(struct irq_desc * desc)505 static bool irq_may_run(struct irq_desc *desc)
506 {
507 	unsigned int mask = IRQD_IRQ_INPROGRESS | IRQD_WAKEUP_ARMED;
508 
509 	/*
510 	 * If the interrupt is not in progress and is not an armed
511 	 * wakeup interrupt, proceed.
512 	 */
513 	if (!irqd_has_set(&desc->irq_data, mask))
514 		return true;
515 
516 	/*
517 	 * If the interrupt is an armed wakeup source, mark it pending
518 	 * and suspended, disable it and notify the pm core about the
519 	 * event.
520 	 */
521 	if (irq_pm_check_wakeup(desc))
522 		return false;
523 
524 	/*
525 	 * Handle a potential concurrent poll on a different core.
526 	 */
527 	return irq_check_poll(desc);
528 }
529 
530 /**
531  *	handle_simple_irq - Simple and software-decoded IRQs.
532  *	@desc:	the interrupt description structure for this irq
533  *
534  *	Simple interrupts are either sent from a demultiplexing interrupt
535  *	handler or come from hardware, where no interrupt hardware control
536  *	is necessary.
537  *
538  *	Note: The caller is expected to handle the ack, clear, mask and
539  *	unmask issues if necessary.
540  */
handle_simple_irq(struct irq_desc * desc)541 void handle_simple_irq(struct irq_desc *desc)
542 {
543 	raw_spin_lock(&desc->lock);
544 
545 	if (!irq_may_run(desc))
546 		goto out_unlock;
547 
548 	desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
549 
550 	if (unlikely(!desc->action || irqd_irq_disabled(&desc->irq_data))) {
551 		desc->istate |= IRQS_PENDING;
552 		goto out_unlock;
553 	}
554 
555 	kstat_incr_irqs_this_cpu(desc);
556 	handle_irq_event(desc);
557 
558 out_unlock:
559 	raw_spin_unlock(&desc->lock);
560 }
561 EXPORT_SYMBOL_GPL(handle_simple_irq);
562 
563 /**
564  *	handle_untracked_irq - Simple and software-decoded IRQs.
565  *	@desc:	the interrupt description structure for this irq
566  *
567  *	Untracked interrupts are sent from a demultiplexing interrupt
568  *	handler when the demultiplexer does not know which device it its
569  *	multiplexed irq domain generated the interrupt. IRQ's handled
570  *	through here are not subjected to stats tracking, randomness, or
571  *	spurious interrupt detection.
572  *
573  *	Note: Like handle_simple_irq, the caller is expected to handle
574  *	the ack, clear, mask and unmask issues if necessary.
575  */
handle_untracked_irq(struct irq_desc * desc)576 void handle_untracked_irq(struct irq_desc *desc)
577 {
578 	unsigned int flags = 0;
579 
580 	raw_spin_lock(&desc->lock);
581 
582 	if (!irq_may_run(desc))
583 		goto out_unlock;
584 
585 	desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
586 
587 	if (unlikely(!desc->action || irqd_irq_disabled(&desc->irq_data))) {
588 		desc->istate |= IRQS_PENDING;
589 		goto out_unlock;
590 	}
591 
592 	desc->istate &= ~IRQS_PENDING;
593 	irqd_set(&desc->irq_data, IRQD_IRQ_INPROGRESS);
594 	raw_spin_unlock(&desc->lock);
595 
596 	__handle_irq_event_percpu(desc, &flags);
597 
598 	raw_spin_lock(&desc->lock);
599 	irqd_clear(&desc->irq_data, IRQD_IRQ_INPROGRESS);
600 
601 out_unlock:
602 	raw_spin_unlock(&desc->lock);
603 }
604 EXPORT_SYMBOL_GPL(handle_untracked_irq);
605 
606 /*
607  * Called unconditionally from handle_level_irq() and only for oneshot
608  * interrupts from handle_fasteoi_irq()
609  */
cond_unmask_irq(struct irq_desc * desc)610 static void cond_unmask_irq(struct irq_desc *desc)
611 {
612 	/*
613 	 * We need to unmask in the following cases:
614 	 * - Standard level irq (IRQF_ONESHOT is not set)
615 	 * - Oneshot irq which did not wake the thread (caused by a
616 	 *   spurious interrupt or a primary handler handling it
617 	 *   completely).
618 	 */
619 	if (!irqd_irq_disabled(&desc->irq_data) &&
620 	    irqd_irq_masked(&desc->irq_data) && !desc->threads_oneshot)
621 		unmask_irq(desc);
622 }
623 
624 /**
625  *	handle_level_irq - Level type irq handler
626  *	@desc:	the interrupt description structure for this irq
627  *
628  *	Level type interrupts are active as long as the hardware line has
629  *	the active level. This may require to mask the interrupt and unmask
630  *	it after the associated handler has acknowledged the device, so the
631  *	interrupt line is back to inactive.
632  */
handle_level_irq(struct irq_desc * desc)633 void handle_level_irq(struct irq_desc *desc)
634 {
635 	raw_spin_lock(&desc->lock);
636 	mask_ack_irq(desc);
637 
638 	if (!irq_may_run(desc))
639 		goto out_unlock;
640 
641 	desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
642 
643 	/*
644 	 * If its disabled or no action available
645 	 * keep it masked and get out of here
646 	 */
647 	if (unlikely(!desc->action || irqd_irq_disabled(&desc->irq_data))) {
648 		desc->istate |= IRQS_PENDING;
649 		goto out_unlock;
650 	}
651 
652 	kstat_incr_irqs_this_cpu(desc);
653 	handle_irq_event(desc);
654 
655 	cond_unmask_irq(desc);
656 
657 out_unlock:
658 	raw_spin_unlock(&desc->lock);
659 }
660 EXPORT_SYMBOL_GPL(handle_level_irq);
661 
cond_unmask_eoi_irq(struct irq_desc * desc,struct irq_chip * chip)662 static void cond_unmask_eoi_irq(struct irq_desc *desc, struct irq_chip *chip)
663 {
664 	if (!(desc->istate & IRQS_ONESHOT)) {
665 		chip->irq_eoi(&desc->irq_data);
666 		return;
667 	}
668 	/*
669 	 * We need to unmask in the following cases:
670 	 * - Oneshot irq which did not wake the thread (caused by a
671 	 *   spurious interrupt or a primary handler handling it
672 	 *   completely).
673 	 */
674 	if (!irqd_irq_disabled(&desc->irq_data) &&
675 	    irqd_irq_masked(&desc->irq_data) && !desc->threads_oneshot) {
676 		chip->irq_eoi(&desc->irq_data);
677 		unmask_irq(desc);
678 	} else if (!(chip->flags & IRQCHIP_EOI_THREADED)) {
679 		chip->irq_eoi(&desc->irq_data);
680 	}
681 }
682 
683 /**
684  *	handle_fasteoi_irq - irq handler for transparent controllers
685  *	@desc:	the interrupt description structure for this irq
686  *
687  *	Only a single callback will be issued to the chip: an ->eoi()
688  *	call when the interrupt has been serviced. This enables support
689  *	for modern forms of interrupt handlers, which handle the flow
690  *	details in hardware, transparently.
691  */
handle_fasteoi_irq(struct irq_desc * desc)692 void handle_fasteoi_irq(struct irq_desc *desc)
693 {
694 	struct irq_chip *chip = desc->irq_data.chip;
695 
696 	raw_spin_lock(&desc->lock);
697 
698 	if (!irq_may_run(desc))
699 		goto out;
700 
701 	desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
702 
703 	/*
704 	 * If its disabled or no action available
705 	 * then mask it and get out of here:
706 	 */
707 	if (unlikely(!desc->action || irqd_irq_disabled(&desc->irq_data))) {
708 		desc->istate |= IRQS_PENDING;
709 		mask_irq(desc);
710 		goto out;
711 	}
712 
713 	kstat_incr_irqs_this_cpu(desc);
714 	if (desc->istate & IRQS_ONESHOT)
715 		mask_irq(desc);
716 
717 	handle_irq_event(desc);
718 
719 	cond_unmask_eoi_irq(desc, chip);
720 
721 	raw_spin_unlock(&desc->lock);
722 	return;
723 out:
724 	if (!(chip->flags & IRQCHIP_EOI_IF_HANDLED))
725 		chip->irq_eoi(&desc->irq_data);
726 	raw_spin_unlock(&desc->lock);
727 }
728 EXPORT_SYMBOL_GPL(handle_fasteoi_irq);
729 
730 /**
731  *	handle_fasteoi_nmi - irq handler for NMI interrupt lines
732  *	@desc:	the interrupt description structure for this irq
733  *
734  *	A simple NMI-safe handler, considering the restrictions
735  *	from request_nmi.
736  *
737  *	Only a single callback will be issued to the chip: an ->eoi()
738  *	call when the interrupt has been serviced. This enables support
739  *	for modern forms of interrupt handlers, which handle the flow
740  *	details in hardware, transparently.
741  */
handle_fasteoi_nmi(struct irq_desc * desc)742 void handle_fasteoi_nmi(struct irq_desc *desc)
743 {
744 	struct irq_chip *chip = irq_desc_get_chip(desc);
745 	struct irqaction *action = desc->action;
746 	unsigned int irq = irq_desc_get_irq(desc);
747 	irqreturn_t res;
748 
749 	__kstat_incr_irqs_this_cpu(desc);
750 
751 	trace_irq_handler_entry(irq, action);
752 	/*
753 	 * NMIs cannot be shared, there is only one action.
754 	 */
755 	res = action->handler(irq, action->dev_id);
756 	trace_irq_handler_exit(irq, action, res);
757 
758 	if (chip->irq_eoi)
759 		chip->irq_eoi(&desc->irq_data);
760 }
761 EXPORT_SYMBOL_GPL(handle_fasteoi_nmi);
762 
763 /**
764  *	handle_edge_irq - edge type IRQ handler
765  *	@desc:	the interrupt description structure for this irq
766  *
767  *	Interrupt occures on the falling and/or rising edge of a hardware
768  *	signal. The occurrence is latched into the irq controller hardware
769  *	and must be acked in order to be reenabled. After the ack another
770  *	interrupt can happen on the same source even before the first one
771  *	is handled by the associated event handler. If this happens it
772  *	might be necessary to disable (mask) the interrupt depending on the
773  *	controller hardware. This requires to reenable the interrupt inside
774  *	of the loop which handles the interrupts which have arrived while
775  *	the handler was running. If all pending interrupts are handled, the
776  *	loop is left.
777  */
handle_edge_irq(struct irq_desc * desc)778 void handle_edge_irq(struct irq_desc *desc)
779 {
780 	raw_spin_lock(&desc->lock);
781 
782 	desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
783 
784 	if (!irq_may_run(desc)) {
785 		desc->istate |= IRQS_PENDING;
786 		mask_ack_irq(desc);
787 		goto out_unlock;
788 	}
789 
790 	/*
791 	 * If its disabled or no action available then mask it and get
792 	 * out of here.
793 	 */
794 	if (irqd_irq_disabled(&desc->irq_data) || !desc->action) {
795 		desc->istate |= IRQS_PENDING;
796 		mask_ack_irq(desc);
797 		goto out_unlock;
798 	}
799 
800 	kstat_incr_irqs_this_cpu(desc);
801 
802 	/* Start handling the irq */
803 	desc->irq_data.chip->irq_ack(&desc->irq_data);
804 
805 	do {
806 		if (unlikely(!desc->action)) {
807 			mask_irq(desc);
808 			goto out_unlock;
809 		}
810 
811 		/*
812 		 * When another irq arrived while we were handling
813 		 * one, we could have masked the irq.
814 		 * Renable it, if it was not disabled in meantime.
815 		 */
816 		if (unlikely(desc->istate & IRQS_PENDING)) {
817 			if (!irqd_irq_disabled(&desc->irq_data) &&
818 			    irqd_irq_masked(&desc->irq_data))
819 				unmask_irq(desc);
820 		}
821 
822 		handle_irq_event(desc);
823 
824 	} while ((desc->istate & IRQS_PENDING) &&
825 		 !irqd_irq_disabled(&desc->irq_data));
826 
827 out_unlock:
828 	raw_spin_unlock(&desc->lock);
829 }
830 EXPORT_SYMBOL(handle_edge_irq);
831 
832 #ifdef CONFIG_IRQ_EDGE_EOI_HANDLER
833 /**
834  *	handle_edge_eoi_irq - edge eoi type IRQ handler
835  *	@desc:	the interrupt description structure for this irq
836  *
837  * Similar as the above handle_edge_irq, but using eoi and w/o the
838  * mask/unmask logic.
839  */
handle_edge_eoi_irq(struct irq_desc * desc)840 void handle_edge_eoi_irq(struct irq_desc *desc)
841 {
842 	struct irq_chip *chip = irq_desc_get_chip(desc);
843 
844 	raw_spin_lock(&desc->lock);
845 
846 	desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
847 
848 	if (!irq_may_run(desc)) {
849 		desc->istate |= IRQS_PENDING;
850 		goto out_eoi;
851 	}
852 
853 	/*
854 	 * If its disabled or no action available then mask it and get
855 	 * out of here.
856 	 */
857 	if (irqd_irq_disabled(&desc->irq_data) || !desc->action) {
858 		desc->istate |= IRQS_PENDING;
859 		goto out_eoi;
860 	}
861 
862 	kstat_incr_irqs_this_cpu(desc);
863 
864 	do {
865 		if (unlikely(!desc->action))
866 			goto out_eoi;
867 
868 		handle_irq_event(desc);
869 
870 	} while ((desc->istate & IRQS_PENDING) &&
871 		 !irqd_irq_disabled(&desc->irq_data));
872 
873 out_eoi:
874 	chip->irq_eoi(&desc->irq_data);
875 	raw_spin_unlock(&desc->lock);
876 }
877 #endif
878 
879 /**
880  *	handle_percpu_irq - Per CPU local irq handler
881  *	@desc:	the interrupt description structure for this irq
882  *
883  *	Per CPU interrupts on SMP machines without locking requirements
884  */
handle_percpu_irq(struct irq_desc * desc)885 void handle_percpu_irq(struct irq_desc *desc)
886 {
887 	struct irq_chip *chip = irq_desc_get_chip(desc);
888 
889 	/*
890 	 * PER CPU interrupts are not serialized. Do not touch
891 	 * desc->tot_count.
892 	 */
893 	__kstat_incr_irqs_this_cpu(desc);
894 
895 	if (chip->irq_ack)
896 		chip->irq_ack(&desc->irq_data);
897 
898 	handle_irq_event_percpu(desc);
899 
900 	if (chip->irq_eoi)
901 		chip->irq_eoi(&desc->irq_data);
902 }
903 
904 /**
905  * handle_percpu_devid_irq - Per CPU local irq handler with per cpu dev ids
906  * @desc:	the interrupt description structure for this irq
907  *
908  * Per CPU interrupts on SMP machines without locking requirements. Same as
909  * handle_percpu_irq() above but with the following extras:
910  *
911  * action->percpu_dev_id is a pointer to percpu variables which
912  * contain the real device id for the cpu on which this handler is
913  * called
914  */
handle_percpu_devid_irq(struct irq_desc * desc)915 void handle_percpu_devid_irq(struct irq_desc *desc)
916 {
917 	struct irq_chip *chip = irq_desc_get_chip(desc);
918 	struct irqaction *action = desc->action;
919 	unsigned int irq = irq_desc_get_irq(desc);
920 	irqreturn_t res;
921 
922 	/*
923 	 * PER CPU interrupts are not serialized. Do not touch
924 	 * desc->tot_count.
925 	 */
926 	__kstat_incr_irqs_this_cpu(desc);
927 
928 	if (chip->irq_ack)
929 		chip->irq_ack(&desc->irq_data);
930 
931 	if (likely(action)) {
932 		trace_irq_handler_entry(irq, action);
933 		res = action->handler(irq, raw_cpu_ptr(action->percpu_dev_id));
934 		trace_irq_handler_exit(irq, action, res);
935 	} else {
936 		unsigned int cpu = smp_processor_id();
937 		bool enabled = cpumask_test_cpu(cpu, desc->percpu_enabled);
938 
939 		if (enabled)
940 			irq_percpu_disable(desc, cpu);
941 
942 		pr_err_once("Spurious%s percpu IRQ%u on CPU%u\n",
943 			    enabled ? " and unmasked" : "", irq, cpu);
944 	}
945 
946 	if (chip->irq_eoi)
947 		chip->irq_eoi(&desc->irq_data);
948 }
949 
950 /**
951  * handle_percpu_devid_fasteoi_ipi - Per CPU local IPI handler with per cpu
952  *				     dev ids
953  * @desc:	the interrupt description structure for this irq
954  *
955  * The biggest difference with the IRQ version is that the interrupt is
956  * EOIed early, as the IPI could result in a context switch, and we need to
957  * make sure the IPI can fire again. We also assume that the arch code has
958  * registered an action. If not, we are positively doomed.
959  */
handle_percpu_devid_fasteoi_ipi(struct irq_desc * desc)960 void handle_percpu_devid_fasteoi_ipi(struct irq_desc *desc)
961 {
962 	struct irq_chip *chip = irq_desc_get_chip(desc);
963 	struct irqaction *action = desc->action;
964 	unsigned int irq = irq_desc_get_irq(desc);
965 	irqreturn_t res;
966 
967 	__kstat_incr_irqs_this_cpu(desc);
968 
969 	if (chip->irq_eoi)
970 		chip->irq_eoi(&desc->irq_data);
971 
972 	trace_irq_handler_entry(irq, action);
973 	res = action->handler(irq, raw_cpu_ptr(action->percpu_dev_id));
974 	trace_irq_handler_exit(irq, action, res);
975 }
976 
977 /**
978  * handle_percpu_devid_fasteoi_nmi - Per CPU local NMI handler with per cpu
979  *				     dev ids
980  * @desc:	the interrupt description structure for this irq
981  *
982  * Similar to handle_fasteoi_nmi, but handling the dev_id cookie
983  * as a percpu pointer.
984  */
handle_percpu_devid_fasteoi_nmi(struct irq_desc * desc)985 void handle_percpu_devid_fasteoi_nmi(struct irq_desc *desc)
986 {
987 	struct irq_chip *chip = irq_desc_get_chip(desc);
988 	struct irqaction *action = desc->action;
989 	unsigned int irq = irq_desc_get_irq(desc);
990 	irqreturn_t res;
991 
992 	__kstat_incr_irqs_this_cpu(desc);
993 
994 	trace_irq_handler_entry(irq, action);
995 	res = action->handler(irq, raw_cpu_ptr(action->percpu_dev_id));
996 	trace_irq_handler_exit(irq, action, res);
997 
998 	if (chip->irq_eoi)
999 		chip->irq_eoi(&desc->irq_data);
1000 }
1001 
1002 static void
__irq_do_set_handler(struct irq_desc * desc,irq_flow_handler_t handle,int is_chained,const char * name)1003 __irq_do_set_handler(struct irq_desc *desc, irq_flow_handler_t handle,
1004 		     int is_chained, const char *name)
1005 {
1006 	if (!handle) {
1007 		handle = handle_bad_irq;
1008 	} else {
1009 		struct irq_data *irq_data = &desc->irq_data;
1010 #ifdef CONFIG_IRQ_DOMAIN_HIERARCHY
1011 		/*
1012 		 * With hierarchical domains we might run into a
1013 		 * situation where the outermost chip is not yet set
1014 		 * up, but the inner chips are there.  Instead of
1015 		 * bailing we install the handler, but obviously we
1016 		 * cannot enable/startup the interrupt at this point.
1017 		 */
1018 		while (irq_data) {
1019 			if (irq_data->chip != &no_irq_chip)
1020 				break;
1021 			/*
1022 			 * Bail out if the outer chip is not set up
1023 			 * and the interrupt supposed to be started
1024 			 * right away.
1025 			 */
1026 			if (WARN_ON(is_chained))
1027 				return;
1028 			/* Try the parent */
1029 			irq_data = irq_data->parent_data;
1030 		}
1031 #endif
1032 		if (WARN_ON(!irq_data || irq_data->chip == &no_irq_chip))
1033 			return;
1034 	}
1035 
1036 	/* Uninstall? */
1037 	if (handle == handle_bad_irq) {
1038 		if (desc->irq_data.chip != &no_irq_chip)
1039 			mask_ack_irq(desc);
1040 		irq_state_set_disabled(desc);
1041 		if (is_chained)
1042 			desc->action = NULL;
1043 		desc->depth = 1;
1044 	}
1045 	desc->handle_irq = handle;
1046 	desc->name = name;
1047 
1048 	if (handle != handle_bad_irq && is_chained) {
1049 		unsigned int type = irqd_get_trigger_type(&desc->irq_data);
1050 
1051 		/*
1052 		 * We're about to start this interrupt immediately,
1053 		 * hence the need to set the trigger configuration.
1054 		 * But the .set_type callback may have overridden the
1055 		 * flow handler, ignoring that we're dealing with a
1056 		 * chained interrupt. Reset it immediately because we
1057 		 * do know better.
1058 		 */
1059 		if (type != IRQ_TYPE_NONE) {
1060 			__irq_set_trigger(desc, type);
1061 			desc->handle_irq = handle;
1062 		}
1063 
1064 		irq_settings_set_noprobe(desc);
1065 		irq_settings_set_norequest(desc);
1066 		irq_settings_set_nothread(desc);
1067 		desc->action = &chained_action;
1068 		irq_activate_and_startup(desc, IRQ_RESEND);
1069 	}
1070 }
1071 
1072 void
__irq_set_handler(unsigned int irq,irq_flow_handler_t handle,int is_chained,const char * name)1073 __irq_set_handler(unsigned int irq, irq_flow_handler_t handle, int is_chained,
1074 		  const char *name)
1075 {
1076 	unsigned long flags;
1077 	struct irq_desc *desc = irq_get_desc_buslock(irq, &flags, 0);
1078 
1079 	if (!desc)
1080 		return;
1081 
1082 	__irq_do_set_handler(desc, handle, is_chained, name);
1083 	irq_put_desc_busunlock(desc, flags);
1084 }
1085 EXPORT_SYMBOL_GPL(__irq_set_handler);
1086 
1087 void
irq_set_chained_handler_and_data(unsigned int irq,irq_flow_handler_t handle,void * data)1088 irq_set_chained_handler_and_data(unsigned int irq, irq_flow_handler_t handle,
1089 				 void *data)
1090 {
1091 	unsigned long flags;
1092 	struct irq_desc *desc = irq_get_desc_buslock(irq, &flags, 0);
1093 
1094 	if (!desc)
1095 		return;
1096 
1097 	desc->irq_common_data.handler_data = data;
1098 	__irq_do_set_handler(desc, handle, 1, NULL);
1099 
1100 	irq_put_desc_busunlock(desc, flags);
1101 }
1102 EXPORT_SYMBOL_GPL(irq_set_chained_handler_and_data);
1103 
1104 void
irq_set_chip_and_handler_name(unsigned int irq,struct irq_chip * chip,irq_flow_handler_t handle,const char * name)1105 irq_set_chip_and_handler_name(unsigned int irq, struct irq_chip *chip,
1106 			      irq_flow_handler_t handle, const char *name)
1107 {
1108 	irq_set_chip(irq, chip);
1109 	__irq_set_handler(irq, handle, 0, name);
1110 }
1111 EXPORT_SYMBOL_GPL(irq_set_chip_and_handler_name);
1112 
irq_modify_status(unsigned int irq,unsigned long clr,unsigned long set)1113 void irq_modify_status(unsigned int irq, unsigned long clr, unsigned long set)
1114 {
1115 	unsigned long flags, trigger, tmp;
1116 	struct irq_desc *desc = irq_get_desc_lock(irq, &flags, 0);
1117 
1118 	if (!desc)
1119 		return;
1120 
1121 	/*
1122 	 * Warn when a driver sets the no autoenable flag on an already
1123 	 * active interrupt.
1124 	 */
1125 	WARN_ON_ONCE(!desc->depth && (set & _IRQ_NOAUTOEN));
1126 
1127 	irq_settings_clr_and_set(desc, clr, set);
1128 
1129 	trigger = irqd_get_trigger_type(&desc->irq_data);
1130 
1131 	irqd_clear(&desc->irq_data, IRQD_NO_BALANCING | IRQD_PER_CPU |
1132 		   IRQD_TRIGGER_MASK | IRQD_LEVEL | IRQD_MOVE_PCNTXT);
1133 	if (irq_settings_has_no_balance_set(desc))
1134 		irqd_set(&desc->irq_data, IRQD_NO_BALANCING);
1135 	if (irq_settings_is_per_cpu(desc))
1136 		irqd_set(&desc->irq_data, IRQD_PER_CPU);
1137 	if (irq_settings_can_move_pcntxt(desc))
1138 		irqd_set(&desc->irq_data, IRQD_MOVE_PCNTXT);
1139 	if (irq_settings_is_level(desc))
1140 		irqd_set(&desc->irq_data, IRQD_LEVEL);
1141 
1142 	tmp = irq_settings_get_trigger_mask(desc);
1143 	if (tmp != IRQ_TYPE_NONE)
1144 		trigger = tmp;
1145 
1146 	irqd_set(&desc->irq_data, trigger);
1147 
1148 	irq_put_desc_unlock(desc, flags);
1149 }
1150 EXPORT_SYMBOL_GPL(irq_modify_status);
1151 
1152 /**
1153  *	irq_cpu_online - Invoke all irq_cpu_online functions.
1154  *
1155  *	Iterate through all irqs and invoke the chip.irq_cpu_online()
1156  *	for each.
1157  */
irq_cpu_online(void)1158 void irq_cpu_online(void)
1159 {
1160 	struct irq_desc *desc;
1161 	struct irq_chip *chip;
1162 	unsigned long flags;
1163 	unsigned int irq;
1164 
1165 	for_each_active_irq(irq) {
1166 		desc = irq_to_desc(irq);
1167 		if (!desc)
1168 			continue;
1169 
1170 		raw_spin_lock_irqsave(&desc->lock, flags);
1171 
1172 		chip = irq_data_get_irq_chip(&desc->irq_data);
1173 		if (chip && chip->irq_cpu_online &&
1174 		    (!(chip->flags & IRQCHIP_ONOFFLINE_ENABLED) ||
1175 		     !irqd_irq_disabled(&desc->irq_data)))
1176 			chip->irq_cpu_online(&desc->irq_data);
1177 
1178 		raw_spin_unlock_irqrestore(&desc->lock, flags);
1179 	}
1180 }
1181 
1182 /**
1183  *	irq_cpu_offline - Invoke all irq_cpu_offline functions.
1184  *
1185  *	Iterate through all irqs and invoke the chip.irq_cpu_offline()
1186  *	for each.
1187  */
irq_cpu_offline(void)1188 void irq_cpu_offline(void)
1189 {
1190 	struct irq_desc *desc;
1191 	struct irq_chip *chip;
1192 	unsigned long flags;
1193 	unsigned int irq;
1194 
1195 	for_each_active_irq(irq) {
1196 		desc = irq_to_desc(irq);
1197 		if (!desc)
1198 			continue;
1199 
1200 		raw_spin_lock_irqsave(&desc->lock, flags);
1201 
1202 		chip = irq_data_get_irq_chip(&desc->irq_data);
1203 		if (chip && chip->irq_cpu_offline &&
1204 		    (!(chip->flags & IRQCHIP_ONOFFLINE_ENABLED) ||
1205 		     !irqd_irq_disabled(&desc->irq_data)))
1206 			chip->irq_cpu_offline(&desc->irq_data);
1207 
1208 		raw_spin_unlock_irqrestore(&desc->lock, flags);
1209 	}
1210 }
1211 
1212 #ifdef	CONFIG_IRQ_DOMAIN_HIERARCHY
1213 
1214 #ifdef CONFIG_IRQ_FASTEOI_HIERARCHY_HANDLERS
1215 /**
1216  *	handle_fasteoi_ack_irq - irq handler for edge hierarchy
1217  *	stacked on transparent controllers
1218  *
1219  *	@desc:	the interrupt description structure for this irq
1220  *
1221  *	Like handle_fasteoi_irq(), but for use with hierarchy where
1222  *	the irq_chip also needs to have its ->irq_ack() function
1223  *	called.
1224  */
handle_fasteoi_ack_irq(struct irq_desc * desc)1225 void handle_fasteoi_ack_irq(struct irq_desc *desc)
1226 {
1227 	struct irq_chip *chip = desc->irq_data.chip;
1228 
1229 	raw_spin_lock(&desc->lock);
1230 
1231 	if (!irq_may_run(desc))
1232 		goto out;
1233 
1234 	desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
1235 
1236 	/*
1237 	 * If its disabled or no action available
1238 	 * then mask it and get out of here:
1239 	 */
1240 	if (unlikely(!desc->action || irqd_irq_disabled(&desc->irq_data))) {
1241 		desc->istate |= IRQS_PENDING;
1242 		mask_irq(desc);
1243 		goto out;
1244 	}
1245 
1246 	kstat_incr_irqs_this_cpu(desc);
1247 	if (desc->istate & IRQS_ONESHOT)
1248 		mask_irq(desc);
1249 
1250 	/* Start handling the irq */
1251 	desc->irq_data.chip->irq_ack(&desc->irq_data);
1252 
1253 	handle_irq_event(desc);
1254 
1255 	cond_unmask_eoi_irq(desc, chip);
1256 
1257 	raw_spin_unlock(&desc->lock);
1258 	return;
1259 out:
1260 	if (!(chip->flags & IRQCHIP_EOI_IF_HANDLED))
1261 		chip->irq_eoi(&desc->irq_data);
1262 	raw_spin_unlock(&desc->lock);
1263 }
1264 EXPORT_SYMBOL_GPL(handle_fasteoi_ack_irq);
1265 
1266 /**
1267  *	handle_fasteoi_mask_irq - irq handler for level hierarchy
1268  *	stacked on transparent controllers
1269  *
1270  *	@desc:	the interrupt description structure for this irq
1271  *
1272  *	Like handle_fasteoi_irq(), but for use with hierarchy where
1273  *	the irq_chip also needs to have its ->irq_mask_ack() function
1274  *	called.
1275  */
handle_fasteoi_mask_irq(struct irq_desc * desc)1276 void handle_fasteoi_mask_irq(struct irq_desc *desc)
1277 {
1278 	struct irq_chip *chip = desc->irq_data.chip;
1279 
1280 	raw_spin_lock(&desc->lock);
1281 	mask_ack_irq(desc);
1282 
1283 	if (!irq_may_run(desc))
1284 		goto out;
1285 
1286 	desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
1287 
1288 	/*
1289 	 * If its disabled or no action available
1290 	 * then mask it and get out of here:
1291 	 */
1292 	if (unlikely(!desc->action || irqd_irq_disabled(&desc->irq_data))) {
1293 		desc->istate |= IRQS_PENDING;
1294 		mask_irq(desc);
1295 		goto out;
1296 	}
1297 
1298 	kstat_incr_irqs_this_cpu(desc);
1299 	if (desc->istate & IRQS_ONESHOT)
1300 		mask_irq(desc);
1301 
1302 	handle_irq_event(desc);
1303 
1304 	cond_unmask_eoi_irq(desc, chip);
1305 
1306 	raw_spin_unlock(&desc->lock);
1307 	return;
1308 out:
1309 	if (!(chip->flags & IRQCHIP_EOI_IF_HANDLED))
1310 		chip->irq_eoi(&desc->irq_data);
1311 	raw_spin_unlock(&desc->lock);
1312 }
1313 EXPORT_SYMBOL_GPL(handle_fasteoi_mask_irq);
1314 
1315 #endif /* CONFIG_IRQ_FASTEOI_HIERARCHY_HANDLERS */
1316 
1317 /**
1318  * irq_chip_set_parent_state - set the state of a parent interrupt.
1319  *
1320  * @data: Pointer to interrupt specific data
1321  * @which: State to be restored (one of IRQCHIP_STATE_*)
1322  * @val: Value corresponding to @which
1323  *
1324  * Conditional success, if the underlying irqchip does not implement it.
1325  */
irq_chip_set_parent_state(struct irq_data * data,enum irqchip_irq_state which,bool val)1326 int irq_chip_set_parent_state(struct irq_data *data,
1327 			      enum irqchip_irq_state which,
1328 			      bool val)
1329 {
1330 	data = data->parent_data;
1331 
1332 	if (!data || !data->chip->irq_set_irqchip_state)
1333 		return 0;
1334 
1335 	return data->chip->irq_set_irqchip_state(data, which, val);
1336 }
1337 EXPORT_SYMBOL_GPL(irq_chip_set_parent_state);
1338 
1339 /**
1340  * irq_chip_get_parent_state - get the state of a parent interrupt.
1341  *
1342  * @data: Pointer to interrupt specific data
1343  * @which: one of IRQCHIP_STATE_* the caller wants to know
1344  * @state: a pointer to a boolean where the state is to be stored
1345  *
1346  * Conditional success, if the underlying irqchip does not implement it.
1347  */
irq_chip_get_parent_state(struct irq_data * data,enum irqchip_irq_state which,bool * state)1348 int irq_chip_get_parent_state(struct irq_data *data,
1349 			      enum irqchip_irq_state which,
1350 			      bool *state)
1351 {
1352 	data = data->parent_data;
1353 
1354 	if (!data || !data->chip->irq_get_irqchip_state)
1355 		return 0;
1356 
1357 	return data->chip->irq_get_irqchip_state(data, which, state);
1358 }
1359 EXPORT_SYMBOL_GPL(irq_chip_get_parent_state);
1360 
1361 /**
1362  * irq_chip_enable_parent - Enable the parent interrupt (defaults to unmask if
1363  * NULL)
1364  * @data:	Pointer to interrupt specific data
1365  */
irq_chip_enable_parent(struct irq_data * data)1366 void irq_chip_enable_parent(struct irq_data *data)
1367 {
1368 	data = data->parent_data;
1369 	if (data->chip->irq_enable)
1370 		data->chip->irq_enable(data);
1371 	else
1372 		data->chip->irq_unmask(data);
1373 }
1374 EXPORT_SYMBOL_GPL(irq_chip_enable_parent);
1375 
1376 /**
1377  * irq_chip_disable_parent - Disable the parent interrupt (defaults to mask if
1378  * NULL)
1379  * @data:	Pointer to interrupt specific data
1380  */
irq_chip_disable_parent(struct irq_data * data)1381 void irq_chip_disable_parent(struct irq_data *data)
1382 {
1383 	data = data->parent_data;
1384 	if (data->chip->irq_disable)
1385 		data->chip->irq_disable(data);
1386 	else
1387 		data->chip->irq_mask(data);
1388 }
1389 EXPORT_SYMBOL_GPL(irq_chip_disable_parent);
1390 
1391 /**
1392  * irq_chip_ack_parent - Acknowledge the parent interrupt
1393  * @data:	Pointer to interrupt specific data
1394  */
irq_chip_ack_parent(struct irq_data * data)1395 void irq_chip_ack_parent(struct irq_data *data)
1396 {
1397 	data = data->parent_data;
1398 	data->chip->irq_ack(data);
1399 }
1400 EXPORT_SYMBOL_GPL(irq_chip_ack_parent);
1401 
1402 /**
1403  * irq_chip_mask_parent - Mask the parent interrupt
1404  * @data:	Pointer to interrupt specific data
1405  */
irq_chip_mask_parent(struct irq_data * data)1406 void irq_chip_mask_parent(struct irq_data *data)
1407 {
1408 	data = data->parent_data;
1409 	data->chip->irq_mask(data);
1410 }
1411 EXPORT_SYMBOL_GPL(irq_chip_mask_parent);
1412 
1413 /**
1414  * irq_chip_mask_ack_parent - Mask and acknowledge the parent interrupt
1415  * @data:	Pointer to interrupt specific data
1416  */
irq_chip_mask_ack_parent(struct irq_data * data)1417 void irq_chip_mask_ack_parent(struct irq_data *data)
1418 {
1419 	data = data->parent_data;
1420 	data->chip->irq_mask_ack(data);
1421 }
1422 EXPORT_SYMBOL_GPL(irq_chip_mask_ack_parent);
1423 
1424 /**
1425  * irq_chip_unmask_parent - Unmask the parent interrupt
1426  * @data:	Pointer to interrupt specific data
1427  */
irq_chip_unmask_parent(struct irq_data * data)1428 void irq_chip_unmask_parent(struct irq_data *data)
1429 {
1430 	data = data->parent_data;
1431 	data->chip->irq_unmask(data);
1432 }
1433 EXPORT_SYMBOL_GPL(irq_chip_unmask_parent);
1434 
1435 /**
1436  * irq_chip_eoi_parent - Invoke EOI on the parent interrupt
1437  * @data:	Pointer to interrupt specific data
1438  */
irq_chip_eoi_parent(struct irq_data * data)1439 void irq_chip_eoi_parent(struct irq_data *data)
1440 {
1441 	data = data->parent_data;
1442 	data->chip->irq_eoi(data);
1443 }
1444 EXPORT_SYMBOL_GPL(irq_chip_eoi_parent);
1445 
1446 /**
1447  * irq_chip_set_affinity_parent - Set affinity on the parent interrupt
1448  * @data:	Pointer to interrupt specific data
1449  * @dest:	The affinity mask to set
1450  * @force:	Flag to enforce setting (disable online checks)
1451  *
1452  * Conditinal, as the underlying parent chip might not implement it.
1453  */
irq_chip_set_affinity_parent(struct irq_data * data,const struct cpumask * dest,bool force)1454 int irq_chip_set_affinity_parent(struct irq_data *data,
1455 				 const struct cpumask *dest, bool force)
1456 {
1457 	data = data->parent_data;
1458 	if (data->chip->irq_set_affinity)
1459 		return data->chip->irq_set_affinity(data, dest, force);
1460 
1461 	return -ENOSYS;
1462 }
1463 EXPORT_SYMBOL_GPL(irq_chip_set_affinity_parent);
1464 
1465 /**
1466  * irq_chip_set_type_parent - Set IRQ type on the parent interrupt
1467  * @data:	Pointer to interrupt specific data
1468  * @type:	IRQ_TYPE_{LEVEL,EDGE}_* value - see include/linux/irq.h
1469  *
1470  * Conditional, as the underlying parent chip might not implement it.
1471  */
irq_chip_set_type_parent(struct irq_data * data,unsigned int type)1472 int irq_chip_set_type_parent(struct irq_data *data, unsigned int type)
1473 {
1474 	data = data->parent_data;
1475 
1476 	if (data->chip->irq_set_type)
1477 		return data->chip->irq_set_type(data, type);
1478 
1479 	return -ENOSYS;
1480 }
1481 EXPORT_SYMBOL_GPL(irq_chip_set_type_parent);
1482 
1483 /**
1484  * irq_chip_retrigger_hierarchy - Retrigger an interrupt in hardware
1485  * @data:	Pointer to interrupt specific data
1486  *
1487  * Iterate through the domain hierarchy of the interrupt and check
1488  * whether a hw retrigger function exists. If yes, invoke it.
1489  */
irq_chip_retrigger_hierarchy(struct irq_data * data)1490 int irq_chip_retrigger_hierarchy(struct irq_data *data)
1491 {
1492 	for (data = data->parent_data; data; data = data->parent_data)
1493 		if (data->chip && data->chip->irq_retrigger)
1494 			return data->chip->irq_retrigger(data);
1495 
1496 	return 0;
1497 }
1498 EXPORT_SYMBOL_GPL(irq_chip_retrigger_hierarchy);
1499 
1500 /**
1501  * irq_chip_set_vcpu_affinity_parent - Set vcpu affinity on the parent interrupt
1502  * @data:	Pointer to interrupt specific data
1503  * @vcpu_info:	The vcpu affinity information
1504  */
irq_chip_set_vcpu_affinity_parent(struct irq_data * data,void * vcpu_info)1505 int irq_chip_set_vcpu_affinity_parent(struct irq_data *data, void *vcpu_info)
1506 {
1507 	data = data->parent_data;
1508 	if (data->chip->irq_set_vcpu_affinity)
1509 		return data->chip->irq_set_vcpu_affinity(data, vcpu_info);
1510 
1511 	return -ENOSYS;
1512 }
1513 EXPORT_SYMBOL_GPL(irq_chip_set_vcpu_affinity_parent);
1514 /**
1515  * irq_chip_set_wake_parent - Set/reset wake-up on the parent interrupt
1516  * @data:	Pointer to interrupt specific data
1517  * @on:		Whether to set or reset the wake-up capability of this irq
1518  *
1519  * Conditional, as the underlying parent chip might not implement it.
1520  */
irq_chip_set_wake_parent(struct irq_data * data,unsigned int on)1521 int irq_chip_set_wake_parent(struct irq_data *data, unsigned int on)
1522 {
1523 	data = data->parent_data;
1524 
1525 	if (data->chip->flags & IRQCHIP_SKIP_SET_WAKE)
1526 		return 0;
1527 
1528 	if (data->chip->irq_set_wake)
1529 		return data->chip->irq_set_wake(data, on);
1530 
1531 	return -ENOSYS;
1532 }
1533 EXPORT_SYMBOL_GPL(irq_chip_set_wake_parent);
1534 
1535 /**
1536  * irq_chip_request_resources_parent - Request resources on the parent interrupt
1537  * @data:	Pointer to interrupt specific data
1538  */
irq_chip_request_resources_parent(struct irq_data * data)1539 int irq_chip_request_resources_parent(struct irq_data *data)
1540 {
1541 	data = data->parent_data;
1542 
1543 	if (data->chip->irq_request_resources)
1544 		return data->chip->irq_request_resources(data);
1545 
1546 	return -ENOSYS;
1547 }
1548 EXPORT_SYMBOL_GPL(irq_chip_request_resources_parent);
1549 
1550 /**
1551  * irq_chip_release_resources_parent - Release resources on the parent interrupt
1552  * @data:	Pointer to interrupt specific data
1553  */
irq_chip_release_resources_parent(struct irq_data * data)1554 void irq_chip_release_resources_parent(struct irq_data *data)
1555 {
1556 	data = data->parent_data;
1557 	if (data->chip->irq_release_resources)
1558 		data->chip->irq_release_resources(data);
1559 }
1560 EXPORT_SYMBOL_GPL(irq_chip_release_resources_parent);
1561 #endif
1562 
1563 /**
1564  * irq_chip_compose_msi_msg - Componse msi message for a irq chip
1565  * @data:	Pointer to interrupt specific data
1566  * @msg:	Pointer to the MSI message
1567  *
1568  * For hierarchical domains we find the first chip in the hierarchy
1569  * which implements the irq_compose_msi_msg callback. For non
1570  * hierarchical we use the top level chip.
1571  */
irq_chip_compose_msi_msg(struct irq_data * data,struct msi_msg * msg)1572 int irq_chip_compose_msi_msg(struct irq_data *data, struct msi_msg *msg)
1573 {
1574 	struct irq_data *pos;
1575 
1576 	for (pos = NULL; !pos && data; data = irqd_get_parent_data(data)) {
1577 		if (data->chip && data->chip->irq_compose_msi_msg)
1578 			pos = data;
1579 	}
1580 
1581 	if (!pos)
1582 		return -ENOSYS;
1583 
1584 	pos->chip->irq_compose_msi_msg(pos, msg);
1585 	return 0;
1586 }
1587 
1588 /**
1589  * irq_chip_pm_get - Enable power for an IRQ chip
1590  * @data:	Pointer to interrupt specific data
1591  *
1592  * Enable the power to the IRQ chip referenced by the interrupt data
1593  * structure.
1594  */
irq_chip_pm_get(struct irq_data * data)1595 int irq_chip_pm_get(struct irq_data *data)
1596 {
1597 	int retval;
1598 
1599 	if (IS_ENABLED(CONFIG_PM) && data->chip->parent_device) {
1600 		retval = pm_runtime_get_sync(data->chip->parent_device);
1601 		if (retval < 0) {
1602 			pm_runtime_put_noidle(data->chip->parent_device);
1603 			return retval;
1604 		}
1605 	}
1606 
1607 	return 0;
1608 }
1609 
1610 /**
1611  * irq_chip_pm_put - Disable power for an IRQ chip
1612  * @data:	Pointer to interrupt specific data
1613  *
1614  * Disable the power to the IRQ chip referenced by the interrupt data
1615  * structure, belongs. Note that power will only be disabled, once this
1616  * function has been called for all IRQs that have called irq_chip_pm_get().
1617  */
irq_chip_pm_put(struct irq_data * data)1618 int irq_chip_pm_put(struct irq_data *data)
1619 {
1620 	int retval = 0;
1621 
1622 	if (IS_ENABLED(CONFIG_PM) && data->chip->parent_device)
1623 		retval = pm_runtime_put(data->chip->parent_device);
1624 
1625 	return (retval < 0) ? retval : 0;
1626 }
1627