1 /*
2 * Copyright 2014 IBM Corp.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
8 */
9
10 #include <linux/interrupt.h>
11 #include <linux/workqueue.h>
12 #include <linux/sched.h>
13 #include <linux/wait.h>
14 #include <linux/slab.h>
15 #include <linux/pid.h>
16 #include <asm/cputable.h>
17 #include <misc/cxl.h>
18
19 #include "cxl.h"
20
21 /* XXX: This is implementation specific */
handle_psl_slice_error(struct cxl_context * ctx,u64 dsisr,u64 errstat)22 static irqreturn_t handle_psl_slice_error(struct cxl_context *ctx, u64 dsisr, u64 errstat)
23 {
24 u64 fir1, fir2, fir_slice, serr, afu_debug;
25
26 fir1 = cxl_p1_read(ctx->afu->adapter, CXL_PSL_FIR1);
27 fir2 = cxl_p1_read(ctx->afu->adapter, CXL_PSL_FIR2);
28 fir_slice = cxl_p1n_read(ctx->afu, CXL_PSL_FIR_SLICE_An);
29 serr = cxl_p1n_read(ctx->afu, CXL_PSL_SERR_An);
30 afu_debug = cxl_p1n_read(ctx->afu, CXL_AFU_DEBUG_An);
31
32 dev_crit(&ctx->afu->dev, "PSL ERROR STATUS: 0x%.16llx\n", errstat);
33 dev_crit(&ctx->afu->dev, "PSL_FIR1: 0x%.16llx\n", fir1);
34 dev_crit(&ctx->afu->dev, "PSL_FIR2: 0x%.16llx\n", fir2);
35 dev_crit(&ctx->afu->dev, "PSL_SERR_An: 0x%.16llx\n", serr);
36 dev_crit(&ctx->afu->dev, "PSL_FIR_SLICE_An: 0x%.16llx\n", fir_slice);
37 dev_crit(&ctx->afu->dev, "CXL_PSL_AFU_DEBUG_An: 0x%.16llx\n", afu_debug);
38
39 dev_crit(&ctx->afu->dev, "STOPPING CXL TRACE\n");
40 cxl_stop_trace(ctx->afu->adapter);
41
42 return cxl_ack_irq(ctx, 0, errstat);
43 }
44
cxl_slice_irq_err(int irq,void * data)45 irqreturn_t cxl_slice_irq_err(int irq, void *data)
46 {
47 struct cxl_afu *afu = data;
48 u64 fir_slice, errstat, serr, afu_debug;
49
50 WARN(irq, "CXL SLICE ERROR interrupt %i\n", irq);
51
52 serr = cxl_p1n_read(afu, CXL_PSL_SERR_An);
53 fir_slice = cxl_p1n_read(afu, CXL_PSL_FIR_SLICE_An);
54 errstat = cxl_p2n_read(afu, CXL_PSL_ErrStat_An);
55 afu_debug = cxl_p1n_read(afu, CXL_AFU_DEBUG_An);
56 dev_crit(&afu->dev, "PSL_SERR_An: 0x%.16llx\n", serr);
57 dev_crit(&afu->dev, "PSL_FIR_SLICE_An: 0x%.16llx\n", fir_slice);
58 dev_crit(&afu->dev, "CXL_PSL_ErrStat_An: 0x%.16llx\n", errstat);
59 dev_crit(&afu->dev, "CXL_PSL_AFU_DEBUG_An: 0x%.16llx\n", afu_debug);
60
61 cxl_p1n_write(afu, CXL_PSL_SERR_An, serr);
62
63 return IRQ_HANDLED;
64 }
65
cxl_irq_err(int irq,void * data)66 static irqreturn_t cxl_irq_err(int irq, void *data)
67 {
68 struct cxl *adapter = data;
69 u64 fir1, fir2, err_ivte;
70
71 WARN(1, "CXL ERROR interrupt %i\n", irq);
72
73 err_ivte = cxl_p1_read(adapter, CXL_PSL_ErrIVTE);
74 dev_crit(&adapter->dev, "PSL_ErrIVTE: 0x%.16llx\n", err_ivte);
75
76 dev_crit(&adapter->dev, "STOPPING CXL TRACE\n");
77 cxl_stop_trace(adapter);
78
79 fir1 = cxl_p1_read(adapter, CXL_PSL_FIR1);
80 fir2 = cxl_p1_read(adapter, CXL_PSL_FIR2);
81
82 dev_crit(&adapter->dev, "PSL_FIR1: 0x%.16llx\nPSL_FIR2: 0x%.16llx\n", fir1, fir2);
83
84 return IRQ_HANDLED;
85 }
86
schedule_cxl_fault(struct cxl_context * ctx,u64 dsisr,u64 dar)87 static irqreturn_t schedule_cxl_fault(struct cxl_context *ctx, u64 dsisr, u64 dar)
88 {
89 ctx->dsisr = dsisr;
90 ctx->dar = dar;
91 schedule_work(&ctx->fault_work);
92 return IRQ_HANDLED;
93 }
94
cxl_irq(int irq,void * data)95 static irqreturn_t cxl_irq(int irq, void *data)
96 {
97 struct cxl_context *ctx = data;
98 struct cxl_irq_info irq_info;
99 u64 dsisr, dar;
100 int result;
101
102 if ((result = cxl_get_irq(ctx, &irq_info))) {
103 WARN(1, "Unable to get CXL IRQ Info: %i\n", result);
104 return IRQ_HANDLED;
105 }
106
107 dsisr = irq_info.dsisr;
108 dar = irq_info.dar;
109
110 pr_devel("CXL interrupt %i for afu pe: %i DSISR: %#llx DAR: %#llx\n", irq, ctx->pe, dsisr, dar);
111
112 if (dsisr & CXL_PSL_DSISR_An_DS) {
113 /*
114 * We don't inherently need to sleep to handle this, but we do
115 * need to get a ref to the task's mm, which we can't do from
116 * irq context without the potential for a deadlock since it
117 * takes the task_lock. An alternate option would be to keep a
118 * reference to the task's mm the entire time it has cxl open,
119 * but to do that we need to solve the issue where we hold a
120 * ref to the mm, but the mm can hold a ref to the fd after an
121 * mmap preventing anything from being cleaned up.
122 */
123 pr_devel("Scheduling segment miss handling for later pe: %i\n", ctx->pe);
124 return schedule_cxl_fault(ctx, dsisr, dar);
125 }
126
127 if (dsisr & CXL_PSL_DSISR_An_M)
128 pr_devel("CXL interrupt: PTE not found\n");
129 if (dsisr & CXL_PSL_DSISR_An_P)
130 pr_devel("CXL interrupt: Storage protection violation\n");
131 if (dsisr & CXL_PSL_DSISR_An_A)
132 pr_devel("CXL interrupt: AFU lock access to write through or cache inhibited storage\n");
133 if (dsisr & CXL_PSL_DSISR_An_S)
134 pr_devel("CXL interrupt: Access was afu_wr or afu_zero\n");
135 if (dsisr & CXL_PSL_DSISR_An_K)
136 pr_devel("CXL interrupt: Access not permitted by virtual page class key protection\n");
137
138 if (dsisr & CXL_PSL_DSISR_An_DM) {
139 /*
140 * In some cases we might be able to handle the fault
141 * immediately if hash_page would succeed, but we still need
142 * the task's mm, which as above we can't get without a lock
143 */
144 pr_devel("Scheduling page fault handling for later pe: %i\n", ctx->pe);
145 return schedule_cxl_fault(ctx, dsisr, dar);
146 }
147 if (dsisr & CXL_PSL_DSISR_An_ST)
148 WARN(1, "CXL interrupt: Segment Table PTE not found\n");
149 if (dsisr & CXL_PSL_DSISR_An_UR)
150 pr_devel("CXL interrupt: AURP PTE not found\n");
151 if (dsisr & CXL_PSL_DSISR_An_PE)
152 return handle_psl_slice_error(ctx, dsisr, irq_info.errstat);
153 if (dsisr & CXL_PSL_DSISR_An_AE) {
154 pr_devel("CXL interrupt: AFU Error %.llx\n", irq_info.afu_err);
155
156 if (ctx->pending_afu_err) {
157 /*
158 * This shouldn't happen - the PSL treats these errors
159 * as fatal and will have reset the AFU, so there's not
160 * much point buffering multiple AFU errors.
161 * OTOH if we DO ever see a storm of these come in it's
162 * probably best that we log them somewhere:
163 */
164 dev_err_ratelimited(&ctx->afu->dev, "CXL AFU Error "
165 "undelivered to pe %i: %.llx\n",
166 ctx->pe, irq_info.afu_err);
167 } else {
168 spin_lock(&ctx->lock);
169 ctx->afu_err = irq_info.afu_err;
170 ctx->pending_afu_err = 1;
171 spin_unlock(&ctx->lock);
172
173 wake_up_all(&ctx->wq);
174 }
175
176 cxl_ack_irq(ctx, CXL_PSL_TFC_An_A, 0);
177 return IRQ_HANDLED;
178 }
179 if (dsisr & CXL_PSL_DSISR_An_OC)
180 pr_devel("CXL interrupt: OS Context Warning\n");
181
182 WARN(1, "Unhandled CXL PSL IRQ\n");
183 return IRQ_HANDLED;
184 }
185
cxl_irq_multiplexed(int irq,void * data)186 static irqreturn_t cxl_irq_multiplexed(int irq, void *data)
187 {
188 struct cxl_afu *afu = data;
189 struct cxl_context *ctx;
190 int ph = cxl_p2n_read(afu, CXL_PSL_PEHandle_An) & 0xffff;
191 int ret;
192
193 rcu_read_lock();
194 ctx = idr_find(&afu->contexts_idr, ph);
195 if (ctx) {
196 ret = cxl_irq(irq, ctx);
197 rcu_read_unlock();
198 return ret;
199 }
200 rcu_read_unlock();
201
202 WARN(1, "Unable to demultiplex CXL PSL IRQ\n");
203 return IRQ_HANDLED;
204 }
205
cxl_irq_afu(int irq,void * data)206 static irqreturn_t cxl_irq_afu(int irq, void *data)
207 {
208 struct cxl_context *ctx = data;
209 irq_hw_number_t hwirq = irqd_to_hwirq(irq_get_irq_data(irq));
210 int irq_off, afu_irq = 1;
211 __u16 range;
212 int r;
213
214 for (r = 1; r < CXL_IRQ_RANGES; r++) {
215 irq_off = hwirq - ctx->irqs.offset[r];
216 range = ctx->irqs.range[r];
217 if (irq_off >= 0 && irq_off < range) {
218 afu_irq += irq_off;
219 break;
220 }
221 afu_irq += range;
222 }
223 if (unlikely(r >= CXL_IRQ_RANGES)) {
224 WARN(1, "Recieved AFU IRQ out of range for pe %i (virq %i hwirq %lx)\n",
225 ctx->pe, irq, hwirq);
226 return IRQ_HANDLED;
227 }
228
229 pr_devel("Received AFU interrupt %i for pe: %i (virq %i hwirq %lx)\n",
230 afu_irq, ctx->pe, irq, hwirq);
231
232 if (unlikely(!ctx->irq_bitmap)) {
233 WARN(1, "Recieved AFU IRQ for context with no IRQ bitmap\n");
234 return IRQ_HANDLED;
235 }
236 spin_lock(&ctx->lock);
237 set_bit(afu_irq - 1, ctx->irq_bitmap);
238 ctx->pending_irq = true;
239 spin_unlock(&ctx->lock);
240
241 wake_up_all(&ctx->wq);
242
243 return IRQ_HANDLED;
244 }
245
cxl_map_irq(struct cxl * adapter,irq_hw_number_t hwirq,irq_handler_t handler,void * cookie)246 unsigned int cxl_map_irq(struct cxl *adapter, irq_hw_number_t hwirq,
247 irq_handler_t handler, void *cookie)
248 {
249 unsigned int virq;
250 int result;
251
252 /* IRQ Domain? */
253 virq = irq_create_mapping(NULL, hwirq);
254 if (!virq) {
255 dev_warn(&adapter->dev, "cxl_map_irq: irq_create_mapping failed\n");
256 return 0;
257 }
258
259 cxl_setup_irq(adapter, hwirq, virq);
260
261 pr_devel("hwirq %#lx mapped to virq %u\n", hwirq, virq);
262
263 result = request_irq(virq, handler, 0, "cxl", cookie);
264 if (result) {
265 dev_warn(&adapter->dev, "cxl_map_irq: request_irq failed: %i\n", result);
266 return 0;
267 }
268
269 return virq;
270 }
271
cxl_unmap_irq(unsigned int virq,void * cookie)272 void cxl_unmap_irq(unsigned int virq, void *cookie)
273 {
274 free_irq(virq, cookie);
275 }
276
cxl_register_one_irq(struct cxl * adapter,irq_handler_t handler,void * cookie,irq_hw_number_t * dest_hwirq,unsigned int * dest_virq)277 static int cxl_register_one_irq(struct cxl *adapter,
278 irq_handler_t handler,
279 void *cookie,
280 irq_hw_number_t *dest_hwirq,
281 unsigned int *dest_virq)
282 {
283 int hwirq, virq;
284
285 if ((hwirq = cxl_alloc_one_irq(adapter)) < 0)
286 return hwirq;
287
288 if (!(virq = cxl_map_irq(adapter, hwirq, handler, cookie)))
289 goto err;
290
291 *dest_hwirq = hwirq;
292 *dest_virq = virq;
293
294 return 0;
295
296 err:
297 cxl_release_one_irq(adapter, hwirq);
298 return -ENOMEM;
299 }
300
cxl_register_psl_err_irq(struct cxl * adapter)301 int cxl_register_psl_err_irq(struct cxl *adapter)
302 {
303 int rc;
304
305 if ((rc = cxl_register_one_irq(adapter, cxl_irq_err, adapter,
306 &adapter->err_hwirq,
307 &adapter->err_virq)))
308 return rc;
309
310 cxl_p1_write(adapter, CXL_PSL_ErrIVTE, adapter->err_hwirq & 0xffff);
311
312 return 0;
313 }
314
cxl_release_psl_err_irq(struct cxl * adapter)315 void cxl_release_psl_err_irq(struct cxl *adapter)
316 {
317 cxl_p1_write(adapter, CXL_PSL_ErrIVTE, 0x0000000000000000);
318 cxl_unmap_irq(adapter->err_virq, adapter);
319 cxl_release_one_irq(adapter, adapter->err_hwirq);
320 }
321
cxl_register_serr_irq(struct cxl_afu * afu)322 int cxl_register_serr_irq(struct cxl_afu *afu)
323 {
324 u64 serr;
325 int rc;
326
327 if ((rc = cxl_register_one_irq(afu->adapter, cxl_slice_irq_err, afu,
328 &afu->serr_hwirq,
329 &afu->serr_virq)))
330 return rc;
331
332 serr = cxl_p1n_read(afu, CXL_PSL_SERR_An);
333 serr = (serr & 0x00ffffffffff0000ULL) | (afu->serr_hwirq & 0xffff);
334 cxl_p1n_write(afu, CXL_PSL_SERR_An, serr);
335
336 return 0;
337 }
338
cxl_release_serr_irq(struct cxl_afu * afu)339 void cxl_release_serr_irq(struct cxl_afu *afu)
340 {
341 cxl_p1n_write(afu, CXL_PSL_SERR_An, 0x0000000000000000);
342 cxl_unmap_irq(afu->serr_virq, afu);
343 cxl_release_one_irq(afu->adapter, afu->serr_hwirq);
344 }
345
cxl_register_psl_irq(struct cxl_afu * afu)346 int cxl_register_psl_irq(struct cxl_afu *afu)
347 {
348 return cxl_register_one_irq(afu->adapter, cxl_irq_multiplexed, afu,
349 &afu->psl_hwirq, &afu->psl_virq);
350 }
351
cxl_release_psl_irq(struct cxl_afu * afu)352 void cxl_release_psl_irq(struct cxl_afu *afu)
353 {
354 cxl_unmap_irq(afu->psl_virq, afu);
355 cxl_release_one_irq(afu->adapter, afu->psl_hwirq);
356 }
357
afu_register_irqs(struct cxl_context * ctx,u32 count)358 int afu_register_irqs(struct cxl_context *ctx, u32 count)
359 {
360 irq_hw_number_t hwirq;
361 int rc, r, i;
362
363 if ((rc = cxl_alloc_irq_ranges(&ctx->irqs, ctx->afu->adapter, count)))
364 return rc;
365
366 /* Multiplexed PSL Interrupt */
367 ctx->irqs.offset[0] = ctx->afu->psl_hwirq;
368 ctx->irqs.range[0] = 1;
369
370 ctx->irq_count = count;
371 ctx->irq_bitmap = kcalloc(BITS_TO_LONGS(count),
372 sizeof(*ctx->irq_bitmap), GFP_KERNEL);
373 if (!ctx->irq_bitmap)
374 return -ENOMEM;
375 for (r = 1; r < CXL_IRQ_RANGES; r++) {
376 hwirq = ctx->irqs.offset[r];
377 for (i = 0; i < ctx->irqs.range[r]; hwirq++, i++) {
378 cxl_map_irq(ctx->afu->adapter, hwirq,
379 cxl_irq_afu, ctx);
380 }
381 }
382
383 return 0;
384 }
385
afu_release_irqs(struct cxl_context * ctx)386 void afu_release_irqs(struct cxl_context *ctx)
387 {
388 irq_hw_number_t hwirq;
389 unsigned int virq;
390 int r, i;
391
392 for (r = 1; r < CXL_IRQ_RANGES; r++) {
393 hwirq = ctx->irqs.offset[r];
394 for (i = 0; i < ctx->irqs.range[r]; hwirq++, i++) {
395 virq = irq_find_mapping(NULL, hwirq);
396 if (virq)
397 cxl_unmap_irq(virq, ctx);
398 }
399 }
400
401 cxl_release_irq_ranges(&ctx->irqs, ctx->afu->adapter);
402 }
403