1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Support for adapter interruptions
4 *
5 * Copyright IBM Corp. 1999, 2007
6 * Author(s): Ingo Adlung <adlung@de.ibm.com>
7 * Cornelia Huck <cornelia.huck@de.ibm.com>
8 * Arnd Bergmann <arndb@de.ibm.com>
9 * Peter Oberparleiter <peter.oberparleiter@de.ibm.com>
10 */
11
12 #include <linux/init.h>
13 #include <linux/irq.h>
14 #include <linux/kernel_stat.h>
15 #include <linux/module.h>
16 #include <linux/mutex.h>
17 #include <linux/rculist.h>
18 #include <linux/slab.h>
19 #include <linux/dmapool.h>
20
21 #include <asm/airq.h>
22 #include <asm/isc.h>
23 #include <asm/cio.h>
24
25 #include "cio.h"
26 #include "cio_debug.h"
27 #include "ioasm.h"
28
29 static DEFINE_SPINLOCK(airq_lists_lock);
30 static struct hlist_head airq_lists[MAX_ISC+1];
31
32 static struct dma_pool *airq_iv_cache;
33
34 /**
35 * register_adapter_interrupt() - register adapter interrupt handler
36 * @airq: pointer to adapter interrupt descriptor
37 *
38 * Returns 0 on success, or -EINVAL.
39 */
register_adapter_interrupt(struct airq_struct * airq)40 int register_adapter_interrupt(struct airq_struct *airq)
41 {
42 char dbf_txt[32];
43
44 if (!airq->handler || airq->isc > MAX_ISC)
45 return -EINVAL;
46 if (!airq->lsi_ptr) {
47 airq->lsi_ptr = kzalloc(1, GFP_KERNEL);
48 if (!airq->lsi_ptr)
49 return -ENOMEM;
50 airq->flags |= AIRQ_PTR_ALLOCATED;
51 }
52 if (!airq->lsi_mask)
53 airq->lsi_mask = 0xff;
54 snprintf(dbf_txt, sizeof(dbf_txt), "rairq:%p", airq);
55 CIO_TRACE_EVENT(4, dbf_txt);
56 isc_register(airq->isc);
57 spin_lock(&airq_lists_lock);
58 hlist_add_head_rcu(&airq->list, &airq_lists[airq->isc]);
59 spin_unlock(&airq_lists_lock);
60 return 0;
61 }
62 EXPORT_SYMBOL(register_adapter_interrupt);
63
64 /**
65 * unregister_adapter_interrupt - unregister adapter interrupt handler
66 * @airq: pointer to adapter interrupt descriptor
67 */
unregister_adapter_interrupt(struct airq_struct * airq)68 void unregister_adapter_interrupt(struct airq_struct *airq)
69 {
70 char dbf_txt[32];
71
72 if (hlist_unhashed(&airq->list))
73 return;
74 snprintf(dbf_txt, sizeof(dbf_txt), "urairq:%p", airq);
75 CIO_TRACE_EVENT(4, dbf_txt);
76 spin_lock(&airq_lists_lock);
77 hlist_del_rcu(&airq->list);
78 spin_unlock(&airq_lists_lock);
79 synchronize_rcu();
80 isc_unregister(airq->isc);
81 if (airq->flags & AIRQ_PTR_ALLOCATED) {
82 kfree(airq->lsi_ptr);
83 airq->lsi_ptr = NULL;
84 airq->flags &= ~AIRQ_PTR_ALLOCATED;
85 }
86 }
87 EXPORT_SYMBOL(unregister_adapter_interrupt);
88
do_airq_interrupt(int irq,void * dummy)89 static irqreturn_t do_airq_interrupt(int irq, void *dummy)
90 {
91 struct tpi_info *tpi_info;
92 struct airq_struct *airq;
93 struct hlist_head *head;
94
95 set_cpu_flag(CIF_NOHZ_DELAY);
96 tpi_info = &get_irq_regs()->tpi_info;
97 trace_s390_cio_adapter_int(tpi_info);
98 head = &airq_lists[tpi_info->isc];
99 rcu_read_lock();
100 hlist_for_each_entry_rcu(airq, head, list)
101 if ((*airq->lsi_ptr & airq->lsi_mask) != 0)
102 airq->handler(airq, !tpi_info->directed_irq);
103 rcu_read_unlock();
104
105 return IRQ_HANDLED;
106 }
107
init_airq_interrupts(void)108 void __init init_airq_interrupts(void)
109 {
110 irq_set_chip_and_handler(THIN_INTERRUPT,
111 &dummy_irq_chip, handle_percpu_irq);
112 if (request_irq(THIN_INTERRUPT, do_airq_interrupt, 0, "AIO", NULL))
113 panic("Failed to register AIO interrupt\n");
114 }
115
iv_size(unsigned long bits)116 static inline unsigned long iv_size(unsigned long bits)
117 {
118 return BITS_TO_LONGS(bits) * sizeof(unsigned long);
119 }
120
121 /**
122 * airq_iv_create - create an interrupt vector
123 * @bits: number of bits in the interrupt vector
124 * @flags: allocation flags
125 *
126 * Returns a pointer to an interrupt vector structure
127 */
airq_iv_create(unsigned long bits,unsigned long flags)128 struct airq_iv *airq_iv_create(unsigned long bits, unsigned long flags)
129 {
130 struct airq_iv *iv;
131 unsigned long size;
132
133 iv = kzalloc(sizeof(*iv), GFP_KERNEL);
134 if (!iv)
135 goto out;
136 iv->bits = bits;
137 iv->flags = flags;
138 size = iv_size(bits);
139
140 if (flags & AIRQ_IV_CACHELINE) {
141 if ((cache_line_size() * BITS_PER_BYTE) < bits
142 || !airq_iv_cache)
143 goto out_free;
144
145 iv->vector = dma_pool_zalloc(airq_iv_cache, GFP_KERNEL,
146 &iv->vector_dma);
147 if (!iv->vector)
148 goto out_free;
149 } else {
150 iv->vector = cio_dma_zalloc(size);
151 if (!iv->vector)
152 goto out_free;
153 }
154 if (flags & AIRQ_IV_ALLOC) {
155 iv->avail = kmalloc(size, GFP_KERNEL);
156 if (!iv->avail)
157 goto out_free;
158 memset(iv->avail, 0xff, size);
159 iv->end = 0;
160 } else
161 iv->end = bits;
162 if (flags & AIRQ_IV_BITLOCK) {
163 iv->bitlock = kzalloc(size, GFP_KERNEL);
164 if (!iv->bitlock)
165 goto out_free;
166 }
167 if (flags & AIRQ_IV_PTR) {
168 size = bits * sizeof(unsigned long);
169 iv->ptr = kzalloc(size, GFP_KERNEL);
170 if (!iv->ptr)
171 goto out_free;
172 }
173 if (flags & AIRQ_IV_DATA) {
174 size = bits * sizeof(unsigned int);
175 iv->data = kzalloc(size, GFP_KERNEL);
176 if (!iv->data)
177 goto out_free;
178 }
179 spin_lock_init(&iv->lock);
180 return iv;
181
182 out_free:
183 kfree(iv->ptr);
184 kfree(iv->bitlock);
185 kfree(iv->avail);
186 if (iv->flags & AIRQ_IV_CACHELINE && iv->vector)
187 dma_pool_free(airq_iv_cache, iv->vector, iv->vector_dma);
188 else
189 cio_dma_free(iv->vector, size);
190 kfree(iv);
191 out:
192 return NULL;
193 }
194 EXPORT_SYMBOL(airq_iv_create);
195
196 /**
197 * airq_iv_release - release an interrupt vector
198 * @iv: pointer to interrupt vector structure
199 */
airq_iv_release(struct airq_iv * iv)200 void airq_iv_release(struct airq_iv *iv)
201 {
202 kfree(iv->data);
203 kfree(iv->ptr);
204 kfree(iv->bitlock);
205 if (iv->flags & AIRQ_IV_CACHELINE)
206 dma_pool_free(airq_iv_cache, iv->vector, iv->vector_dma);
207 else
208 cio_dma_free(iv->vector, iv_size(iv->bits));
209 kfree(iv->avail);
210 kfree(iv);
211 }
212 EXPORT_SYMBOL(airq_iv_release);
213
214 /**
215 * airq_iv_alloc - allocate irq bits from an interrupt vector
216 * @iv: pointer to an interrupt vector structure
217 * @num: number of consecutive irq bits to allocate
218 *
219 * Returns the bit number of the first irq in the allocated block of irqs,
220 * or -1UL if no bit is available or the AIRQ_IV_ALLOC flag has not been
221 * specified
222 */
airq_iv_alloc(struct airq_iv * iv,unsigned long num)223 unsigned long airq_iv_alloc(struct airq_iv *iv, unsigned long num)
224 {
225 unsigned long bit, i, flags;
226
227 if (!iv->avail || num == 0)
228 return -1UL;
229 spin_lock_irqsave(&iv->lock, flags);
230 bit = find_first_bit_inv(iv->avail, iv->bits);
231 while (bit + num <= iv->bits) {
232 for (i = 1; i < num; i++)
233 if (!test_bit_inv(bit + i, iv->avail))
234 break;
235 if (i >= num) {
236 /* Found a suitable block of irqs */
237 for (i = 0; i < num; i++)
238 clear_bit_inv(bit + i, iv->avail);
239 if (bit + num >= iv->end)
240 iv->end = bit + num + 1;
241 break;
242 }
243 bit = find_next_bit_inv(iv->avail, iv->bits, bit + i + 1);
244 }
245 if (bit + num > iv->bits)
246 bit = -1UL;
247 spin_unlock_irqrestore(&iv->lock, flags);
248 return bit;
249 }
250 EXPORT_SYMBOL(airq_iv_alloc);
251
252 /**
253 * airq_iv_free - free irq bits of an interrupt vector
254 * @iv: pointer to interrupt vector structure
255 * @bit: number of the first irq bit to free
256 * @num: number of consecutive irq bits to free
257 */
airq_iv_free(struct airq_iv * iv,unsigned long bit,unsigned long num)258 void airq_iv_free(struct airq_iv *iv, unsigned long bit, unsigned long num)
259 {
260 unsigned long i, flags;
261
262 if (!iv->avail || num == 0)
263 return;
264 spin_lock_irqsave(&iv->lock, flags);
265 for (i = 0; i < num; i++) {
266 /* Clear (possibly left over) interrupt bit */
267 clear_bit_inv(bit + i, iv->vector);
268 /* Make the bit positions available again */
269 set_bit_inv(bit + i, iv->avail);
270 }
271 if (bit + num >= iv->end) {
272 /* Find new end of bit-field */
273 while (iv->end > 0 && !test_bit_inv(iv->end - 1, iv->avail))
274 iv->end--;
275 }
276 spin_unlock_irqrestore(&iv->lock, flags);
277 }
278 EXPORT_SYMBOL(airq_iv_free);
279
280 /**
281 * airq_iv_scan - scan interrupt vector for non-zero bits
282 * @iv: pointer to interrupt vector structure
283 * @start: bit number to start the search
284 * @end: bit number to end the search
285 *
286 * Returns the bit number of the next non-zero interrupt bit, or
287 * -1UL if the scan completed without finding any more any non-zero bits.
288 */
airq_iv_scan(struct airq_iv * iv,unsigned long start,unsigned long end)289 unsigned long airq_iv_scan(struct airq_iv *iv, unsigned long start,
290 unsigned long end)
291 {
292 unsigned long bit;
293
294 /* Find non-zero bit starting from 'ivs->next'. */
295 bit = find_next_bit_inv(iv->vector, end, start);
296 if (bit >= end)
297 return -1UL;
298 clear_bit_inv(bit, iv->vector);
299 return bit;
300 }
301 EXPORT_SYMBOL(airq_iv_scan);
302
airq_init(void)303 int __init airq_init(void)
304 {
305 airq_iv_cache = dma_pool_create("airq_iv_cache", cio_get_dma_css_dev(),
306 cache_line_size(),
307 cache_line_size(), PAGE_SIZE);
308 if (!airq_iv_cache)
309 return -ENOMEM;
310 return 0;
311 }
312