1 /* i915_irq.c -- IRQ support for the I915 -*- linux-c -*-
2 */
3 /*
4 * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
5 * All Rights Reserved.
6 *
7 * Permission is hereby granted, free of charge, to any person obtaining a
8 * copy of this software and associated documentation files (the
9 * "Software"), to deal in the Software without restriction, including
10 * without limitation the rights to use, copy, modify, merge, publish,
11 * distribute, sub license, and/or sell copies of the Software, and to
12 * permit persons to whom the Software is furnished to do so, subject to
13 * the following conditions:
14 *
15 * The above copyright notice and this permission notice (including the
16 * next paragraph) shall be included in all copies or substantial portions
17 * of the Software.
18 *
19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
20 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
21 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
22 * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
23 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
24 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
25 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
26 *
27 */
28
29 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
30
31 #include <linux/slab.h>
32 #include <linux/sysrq.h>
33
34 #include <drm/drm_drv.h>
35
36 #include "display/intel_display_irq.h"
37 #include "display/intel_display_types.h"
38 #include "display/intel_hotplug.h"
39 #include "display/intel_hotplug_irq.h"
40 #include "display/intel_lpe_audio.h"
41 #include "display/intel_psr_regs.h"
42
43 #include "gt/intel_breadcrumbs.h"
44 #include "gt/intel_gt.h"
45 #include "gt/intel_gt_irq.h"
46 #include "gt/intel_gt_pm_irq.h"
47 #include "gt/intel_gt_regs.h"
48 #include "gt/intel_rps.h"
49
50 #include "i915_driver.h"
51 #include "i915_drv.h"
52 #include "i915_irq.h"
53 #include "i915_reg.h"
54
55 /**
56 * DOC: interrupt handling
57 *
58 * These functions provide the basic support for enabling and disabling the
59 * interrupt handling support. There's a lot more functionality in i915_irq.c
60 * and related files, but that will be described in separate chapters.
61 */
62
63 /*
64 * Interrupt statistic for PMU. Increments the counter only if the
65 * interrupt originated from the GPU so interrupts from a device which
66 * shares the interrupt line are not accounted.
67 */
pmu_irq_stats(struct drm_i915_private * i915,irqreturn_t res)68 static inline void pmu_irq_stats(struct drm_i915_private *i915,
69 irqreturn_t res)
70 {
71 if (unlikely(res != IRQ_HANDLED))
72 return;
73
74 /*
75 * A clever compiler translates that into INC. A not so clever one
76 * should at least prevent store tearing.
77 */
78 WRITE_ONCE(i915->pmu.irq_count, i915->pmu.irq_count + 1);
79 }
80
gen3_irq_reset(struct intel_uncore * uncore,i915_reg_t imr,i915_reg_t iir,i915_reg_t ier)81 void gen3_irq_reset(struct intel_uncore *uncore, i915_reg_t imr,
82 i915_reg_t iir, i915_reg_t ier)
83 {
84 intel_uncore_write(uncore, imr, 0xffffffff);
85 intel_uncore_posting_read(uncore, imr);
86
87 intel_uncore_write(uncore, ier, 0);
88
89 /* IIR can theoretically queue up two events. Be paranoid. */
90 intel_uncore_write(uncore, iir, 0xffffffff);
91 intel_uncore_posting_read(uncore, iir);
92 intel_uncore_write(uncore, iir, 0xffffffff);
93 intel_uncore_posting_read(uncore, iir);
94 }
95
gen2_irq_reset(struct intel_uncore * uncore)96 static void gen2_irq_reset(struct intel_uncore *uncore)
97 {
98 intel_uncore_write16(uncore, GEN2_IMR, 0xffff);
99 intel_uncore_posting_read16(uncore, GEN2_IMR);
100
101 intel_uncore_write16(uncore, GEN2_IER, 0);
102
103 /* IIR can theoretically queue up two events. Be paranoid. */
104 intel_uncore_write16(uncore, GEN2_IIR, 0xffff);
105 intel_uncore_posting_read16(uncore, GEN2_IIR);
106 intel_uncore_write16(uncore, GEN2_IIR, 0xffff);
107 intel_uncore_posting_read16(uncore, GEN2_IIR);
108 }
109
110 /*
111 * We should clear IMR at preinstall/uninstall, and just check at postinstall.
112 */
gen3_assert_iir_is_zero(struct intel_uncore * uncore,i915_reg_t reg)113 void gen3_assert_iir_is_zero(struct intel_uncore *uncore, i915_reg_t reg)
114 {
115 u32 val = intel_uncore_read(uncore, reg);
116
117 if (val == 0)
118 return;
119
120 drm_WARN(&uncore->i915->drm, 1,
121 "Interrupt register 0x%x is not zero: 0x%08x\n",
122 i915_mmio_reg_offset(reg), val);
123 intel_uncore_write(uncore, reg, 0xffffffff);
124 intel_uncore_posting_read(uncore, reg);
125 intel_uncore_write(uncore, reg, 0xffffffff);
126 intel_uncore_posting_read(uncore, reg);
127 }
128
gen2_assert_iir_is_zero(struct intel_uncore * uncore)129 static void gen2_assert_iir_is_zero(struct intel_uncore *uncore)
130 {
131 u16 val = intel_uncore_read16(uncore, GEN2_IIR);
132
133 if (val == 0)
134 return;
135
136 drm_WARN(&uncore->i915->drm, 1,
137 "Interrupt register 0x%x is not zero: 0x%08x\n",
138 i915_mmio_reg_offset(GEN2_IIR), val);
139 intel_uncore_write16(uncore, GEN2_IIR, 0xffff);
140 intel_uncore_posting_read16(uncore, GEN2_IIR);
141 intel_uncore_write16(uncore, GEN2_IIR, 0xffff);
142 intel_uncore_posting_read16(uncore, GEN2_IIR);
143 }
144
gen3_irq_init(struct intel_uncore * uncore,i915_reg_t imr,u32 imr_val,i915_reg_t ier,u32 ier_val,i915_reg_t iir)145 void gen3_irq_init(struct intel_uncore *uncore,
146 i915_reg_t imr, u32 imr_val,
147 i915_reg_t ier, u32 ier_val,
148 i915_reg_t iir)
149 {
150 gen3_assert_iir_is_zero(uncore, iir);
151
152 intel_uncore_write(uncore, ier, ier_val);
153 intel_uncore_write(uncore, imr, imr_val);
154 intel_uncore_posting_read(uncore, imr);
155 }
156
gen2_irq_init(struct intel_uncore * uncore,u32 imr_val,u32 ier_val)157 static void gen2_irq_init(struct intel_uncore *uncore,
158 u32 imr_val, u32 ier_val)
159 {
160 gen2_assert_iir_is_zero(uncore);
161
162 intel_uncore_write16(uncore, GEN2_IER, ier_val);
163 intel_uncore_write16(uncore, GEN2_IMR, imr_val);
164 intel_uncore_posting_read16(uncore, GEN2_IMR);
165 }
166
167 /**
168 * ivb_parity_work - Workqueue called when a parity error interrupt
169 * occurred.
170 * @work: workqueue struct
171 *
172 * Doesn't actually do anything except notify userspace. As a consequence of
173 * this event, userspace should try to remap the bad rows since statistically
174 * it is likely the same row is more likely to go bad again.
175 */
ivb_parity_work(struct work_struct * work)176 static void ivb_parity_work(struct work_struct *work)
177 {
178 struct drm_i915_private *dev_priv =
179 container_of(work, typeof(*dev_priv), l3_parity.error_work);
180 struct intel_gt *gt = to_gt(dev_priv);
181 u32 error_status, row, bank, subbank;
182 char *parity_event[6];
183 u32 misccpctl;
184 u8 slice = 0;
185
186 /* We must turn off DOP level clock gating to access the L3 registers.
187 * In order to prevent a get/put style interface, acquire struct mutex
188 * any time we access those registers.
189 */
190 mutex_lock(&dev_priv->drm.struct_mutex);
191
192 /* If we've screwed up tracking, just let the interrupt fire again */
193 if (drm_WARN_ON(&dev_priv->drm, !dev_priv->l3_parity.which_slice))
194 goto out;
195
196 misccpctl = intel_uncore_rmw(&dev_priv->uncore, GEN7_MISCCPCTL,
197 GEN7_DOP_CLOCK_GATE_ENABLE, 0);
198 intel_uncore_posting_read(&dev_priv->uncore, GEN7_MISCCPCTL);
199
200 while ((slice = ffs(dev_priv->l3_parity.which_slice)) != 0) {
201 i915_reg_t reg;
202
203 slice--;
204 if (drm_WARN_ON_ONCE(&dev_priv->drm,
205 slice >= NUM_L3_SLICES(dev_priv)))
206 break;
207
208 dev_priv->l3_parity.which_slice &= ~(1<<slice);
209
210 reg = GEN7_L3CDERRST1(slice);
211
212 error_status = intel_uncore_read(&dev_priv->uncore, reg);
213 row = GEN7_PARITY_ERROR_ROW(error_status);
214 bank = GEN7_PARITY_ERROR_BANK(error_status);
215 subbank = GEN7_PARITY_ERROR_SUBBANK(error_status);
216
217 intel_uncore_write(&dev_priv->uncore, reg, GEN7_PARITY_ERROR_VALID | GEN7_L3CDERRST1_ENABLE);
218 intel_uncore_posting_read(&dev_priv->uncore, reg);
219
220 parity_event[0] = I915_L3_PARITY_UEVENT "=1";
221 parity_event[1] = kasprintf(GFP_KERNEL, "ROW=%d", row);
222 parity_event[2] = kasprintf(GFP_KERNEL, "BANK=%d", bank);
223 parity_event[3] = kasprintf(GFP_KERNEL, "SUBBANK=%d", subbank);
224 parity_event[4] = kasprintf(GFP_KERNEL, "SLICE=%d", slice);
225 parity_event[5] = NULL;
226
227 kobject_uevent_env(&dev_priv->drm.primary->kdev->kobj,
228 KOBJ_CHANGE, parity_event);
229
230 drm_dbg(&dev_priv->drm,
231 "Parity error: Slice = %d, Row = %d, Bank = %d, Sub bank = %d.\n",
232 slice, row, bank, subbank);
233
234 kfree(parity_event[4]);
235 kfree(parity_event[3]);
236 kfree(parity_event[2]);
237 kfree(parity_event[1]);
238 }
239
240 intel_uncore_write(&dev_priv->uncore, GEN7_MISCCPCTL, misccpctl);
241
242 out:
243 drm_WARN_ON(&dev_priv->drm, dev_priv->l3_parity.which_slice);
244 spin_lock_irq(gt->irq_lock);
245 gen5_gt_enable_irq(gt, GT_PARITY_ERROR(dev_priv));
246 spin_unlock_irq(gt->irq_lock);
247
248 mutex_unlock(&dev_priv->drm.struct_mutex);
249 }
250
valleyview_irq_handler(int irq,void * arg)251 static irqreturn_t valleyview_irq_handler(int irq, void *arg)
252 {
253 struct drm_i915_private *dev_priv = arg;
254 irqreturn_t ret = IRQ_NONE;
255
256 if (!intel_irqs_enabled(dev_priv))
257 return IRQ_NONE;
258
259 /* IRQs are synced during runtime_suspend, we don't require a wakeref */
260 disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
261
262 do {
263 u32 iir, gt_iir, pm_iir;
264 u32 pipe_stats[I915_MAX_PIPES] = {};
265 u32 hotplug_status = 0;
266 u32 ier = 0;
267
268 gt_iir = intel_uncore_read(&dev_priv->uncore, GTIIR);
269 pm_iir = intel_uncore_read(&dev_priv->uncore, GEN6_PMIIR);
270 iir = intel_uncore_read(&dev_priv->uncore, VLV_IIR);
271
272 if (gt_iir == 0 && pm_iir == 0 && iir == 0)
273 break;
274
275 ret = IRQ_HANDLED;
276
277 /*
278 * Theory on interrupt generation, based on empirical evidence:
279 *
280 * x = ((VLV_IIR & VLV_IER) ||
281 * (((GT_IIR & GT_IER) || (GEN6_PMIIR & GEN6_PMIER)) &&
282 * (VLV_MASTER_IER & MASTER_INTERRUPT_ENABLE)));
283 *
284 * A CPU interrupt will only be raised when 'x' has a 0->1 edge.
285 * Hence we clear MASTER_INTERRUPT_ENABLE and VLV_IER to
286 * guarantee the CPU interrupt will be raised again even if we
287 * don't end up clearing all the VLV_IIR, GT_IIR, GEN6_PMIIR
288 * bits this time around.
289 */
290 intel_uncore_write(&dev_priv->uncore, VLV_MASTER_IER, 0);
291 ier = intel_uncore_rmw(&dev_priv->uncore, VLV_IER, ~0, 0);
292
293 if (gt_iir)
294 intel_uncore_write(&dev_priv->uncore, GTIIR, gt_iir);
295 if (pm_iir)
296 intel_uncore_write(&dev_priv->uncore, GEN6_PMIIR, pm_iir);
297
298 if (iir & I915_DISPLAY_PORT_INTERRUPT)
299 hotplug_status = i9xx_hpd_irq_ack(dev_priv);
300
301 /* Call regardless, as some status bits might not be
302 * signalled in iir */
303 i9xx_pipestat_irq_ack(dev_priv, iir, pipe_stats);
304
305 if (iir & (I915_LPE_PIPE_A_INTERRUPT |
306 I915_LPE_PIPE_B_INTERRUPT))
307 intel_lpe_audio_irq_handler(dev_priv);
308
309 /*
310 * VLV_IIR is single buffered, and reflects the level
311 * from PIPESTAT/PORT_HOTPLUG_STAT, hence clear it last.
312 */
313 if (iir)
314 intel_uncore_write(&dev_priv->uncore, VLV_IIR, iir);
315
316 intel_uncore_write(&dev_priv->uncore, VLV_IER, ier);
317 intel_uncore_write(&dev_priv->uncore, VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE);
318
319 if (gt_iir)
320 gen6_gt_irq_handler(to_gt(dev_priv), gt_iir);
321 if (pm_iir)
322 gen6_rps_irq_handler(&to_gt(dev_priv)->rps, pm_iir);
323
324 if (hotplug_status)
325 i9xx_hpd_irq_handler(dev_priv, hotplug_status);
326
327 valleyview_pipestat_irq_handler(dev_priv, pipe_stats);
328 } while (0);
329
330 pmu_irq_stats(dev_priv, ret);
331
332 enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
333
334 return ret;
335 }
336
cherryview_irq_handler(int irq,void * arg)337 static irqreturn_t cherryview_irq_handler(int irq, void *arg)
338 {
339 struct drm_i915_private *dev_priv = arg;
340 irqreturn_t ret = IRQ_NONE;
341
342 if (!intel_irqs_enabled(dev_priv))
343 return IRQ_NONE;
344
345 /* IRQs are synced during runtime_suspend, we don't require a wakeref */
346 disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
347
348 do {
349 u32 master_ctl, iir;
350 u32 pipe_stats[I915_MAX_PIPES] = {};
351 u32 hotplug_status = 0;
352 u32 ier = 0;
353
354 master_ctl = intel_uncore_read(&dev_priv->uncore, GEN8_MASTER_IRQ) & ~GEN8_MASTER_IRQ_CONTROL;
355 iir = intel_uncore_read(&dev_priv->uncore, VLV_IIR);
356
357 if (master_ctl == 0 && iir == 0)
358 break;
359
360 ret = IRQ_HANDLED;
361
362 /*
363 * Theory on interrupt generation, based on empirical evidence:
364 *
365 * x = ((VLV_IIR & VLV_IER) ||
366 * ((GEN8_MASTER_IRQ & ~GEN8_MASTER_IRQ_CONTROL) &&
367 * (GEN8_MASTER_IRQ & GEN8_MASTER_IRQ_CONTROL)));
368 *
369 * A CPU interrupt will only be raised when 'x' has a 0->1 edge.
370 * Hence we clear GEN8_MASTER_IRQ_CONTROL and VLV_IER to
371 * guarantee the CPU interrupt will be raised again even if we
372 * don't end up clearing all the VLV_IIR and GEN8_MASTER_IRQ_CONTROL
373 * bits this time around.
374 */
375 intel_uncore_write(&dev_priv->uncore, GEN8_MASTER_IRQ, 0);
376 ier = intel_uncore_rmw(&dev_priv->uncore, VLV_IER, ~0, 0);
377
378 gen8_gt_irq_handler(to_gt(dev_priv), master_ctl);
379
380 if (iir & I915_DISPLAY_PORT_INTERRUPT)
381 hotplug_status = i9xx_hpd_irq_ack(dev_priv);
382
383 /* Call regardless, as some status bits might not be
384 * signalled in iir */
385 i9xx_pipestat_irq_ack(dev_priv, iir, pipe_stats);
386
387 if (iir & (I915_LPE_PIPE_A_INTERRUPT |
388 I915_LPE_PIPE_B_INTERRUPT |
389 I915_LPE_PIPE_C_INTERRUPT))
390 intel_lpe_audio_irq_handler(dev_priv);
391
392 /*
393 * VLV_IIR is single buffered, and reflects the level
394 * from PIPESTAT/PORT_HOTPLUG_STAT, hence clear it last.
395 */
396 if (iir)
397 intel_uncore_write(&dev_priv->uncore, VLV_IIR, iir);
398
399 intel_uncore_write(&dev_priv->uncore, VLV_IER, ier);
400 intel_uncore_write(&dev_priv->uncore, GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
401
402 if (hotplug_status)
403 i9xx_hpd_irq_handler(dev_priv, hotplug_status);
404
405 valleyview_pipestat_irq_handler(dev_priv, pipe_stats);
406 } while (0);
407
408 pmu_irq_stats(dev_priv, ret);
409
410 enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
411
412 return ret;
413 }
414
415 /*
416 * To handle irqs with the minimum potential races with fresh interrupts, we:
417 * 1 - Disable Master Interrupt Control.
418 * 2 - Find the source(s) of the interrupt.
419 * 3 - Clear the Interrupt Identity bits (IIR).
420 * 4 - Process the interrupt(s) that had bits set in the IIRs.
421 * 5 - Re-enable Master Interrupt Control.
422 */
ilk_irq_handler(int irq,void * arg)423 static irqreturn_t ilk_irq_handler(int irq, void *arg)
424 {
425 struct drm_i915_private *i915 = arg;
426 void __iomem * const regs = intel_uncore_regs(&i915->uncore);
427 u32 de_iir, gt_iir, de_ier, sde_ier = 0;
428 irqreturn_t ret = IRQ_NONE;
429
430 if (unlikely(!intel_irqs_enabled(i915)))
431 return IRQ_NONE;
432
433 /* IRQs are synced during runtime_suspend, we don't require a wakeref */
434 disable_rpm_wakeref_asserts(&i915->runtime_pm);
435
436 /* disable master interrupt before clearing iir */
437 de_ier = raw_reg_read(regs, DEIER);
438 raw_reg_write(regs, DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL);
439
440 /* Disable south interrupts. We'll only write to SDEIIR once, so further
441 * interrupts will will be stored on its back queue, and then we'll be
442 * able to process them after we restore SDEIER (as soon as we restore
443 * it, we'll get an interrupt if SDEIIR still has something to process
444 * due to its back queue). */
445 if (!HAS_PCH_NOP(i915)) {
446 sde_ier = raw_reg_read(regs, SDEIER);
447 raw_reg_write(regs, SDEIER, 0);
448 }
449
450 /* Find, clear, then process each source of interrupt */
451
452 gt_iir = raw_reg_read(regs, GTIIR);
453 if (gt_iir) {
454 raw_reg_write(regs, GTIIR, gt_iir);
455 if (GRAPHICS_VER(i915) >= 6)
456 gen6_gt_irq_handler(to_gt(i915), gt_iir);
457 else
458 gen5_gt_irq_handler(to_gt(i915), gt_iir);
459 ret = IRQ_HANDLED;
460 }
461
462 de_iir = raw_reg_read(regs, DEIIR);
463 if (de_iir) {
464 raw_reg_write(regs, DEIIR, de_iir);
465 if (DISPLAY_VER(i915) >= 7)
466 ivb_display_irq_handler(i915, de_iir);
467 else
468 ilk_display_irq_handler(i915, de_iir);
469 ret = IRQ_HANDLED;
470 }
471
472 if (GRAPHICS_VER(i915) >= 6) {
473 u32 pm_iir = raw_reg_read(regs, GEN6_PMIIR);
474 if (pm_iir) {
475 raw_reg_write(regs, GEN6_PMIIR, pm_iir);
476 gen6_rps_irq_handler(&to_gt(i915)->rps, pm_iir);
477 ret = IRQ_HANDLED;
478 }
479 }
480
481 raw_reg_write(regs, DEIER, de_ier);
482 if (sde_ier)
483 raw_reg_write(regs, SDEIER, sde_ier);
484
485 pmu_irq_stats(i915, ret);
486
487 /* IRQs are synced during runtime_suspend, we don't require a wakeref */
488 enable_rpm_wakeref_asserts(&i915->runtime_pm);
489
490 return ret;
491 }
492
gen8_master_intr_disable(void __iomem * const regs)493 static inline u32 gen8_master_intr_disable(void __iomem * const regs)
494 {
495 raw_reg_write(regs, GEN8_MASTER_IRQ, 0);
496
497 /*
498 * Now with master disabled, get a sample of level indications
499 * for this interrupt. Indications will be cleared on related acks.
500 * New indications can and will light up during processing,
501 * and will generate new interrupt after enabling master.
502 */
503 return raw_reg_read(regs, GEN8_MASTER_IRQ);
504 }
505
gen8_master_intr_enable(void __iomem * const regs)506 static inline void gen8_master_intr_enable(void __iomem * const regs)
507 {
508 raw_reg_write(regs, GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
509 }
510
gen8_irq_handler(int irq,void * arg)511 static irqreturn_t gen8_irq_handler(int irq, void *arg)
512 {
513 struct drm_i915_private *dev_priv = arg;
514 void __iomem * const regs = intel_uncore_regs(&dev_priv->uncore);
515 u32 master_ctl;
516
517 if (!intel_irqs_enabled(dev_priv))
518 return IRQ_NONE;
519
520 master_ctl = gen8_master_intr_disable(regs);
521 if (!master_ctl) {
522 gen8_master_intr_enable(regs);
523 return IRQ_NONE;
524 }
525
526 /* Find, queue (onto bottom-halves), then clear each source */
527 gen8_gt_irq_handler(to_gt(dev_priv), master_ctl);
528
529 /* IRQs are synced during runtime_suspend, we don't require a wakeref */
530 if (master_ctl & ~GEN8_GT_IRQS) {
531 disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
532 gen8_de_irq_handler(dev_priv, master_ctl);
533 enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
534 }
535
536 gen8_master_intr_enable(regs);
537
538 pmu_irq_stats(dev_priv, IRQ_HANDLED);
539
540 return IRQ_HANDLED;
541 }
542
gen11_master_intr_disable(void __iomem * const regs)543 static inline u32 gen11_master_intr_disable(void __iomem * const regs)
544 {
545 raw_reg_write(regs, GEN11_GFX_MSTR_IRQ, 0);
546
547 /*
548 * Now with master disabled, get a sample of level indications
549 * for this interrupt. Indications will be cleared on related acks.
550 * New indications can and will light up during processing,
551 * and will generate new interrupt after enabling master.
552 */
553 return raw_reg_read(regs, GEN11_GFX_MSTR_IRQ);
554 }
555
gen11_master_intr_enable(void __iomem * const regs)556 static inline void gen11_master_intr_enable(void __iomem * const regs)
557 {
558 raw_reg_write(regs, GEN11_GFX_MSTR_IRQ, GEN11_MASTER_IRQ);
559 }
560
gen11_irq_handler(int irq,void * arg)561 static irqreturn_t gen11_irq_handler(int irq, void *arg)
562 {
563 struct drm_i915_private *i915 = arg;
564 void __iomem * const regs = intel_uncore_regs(&i915->uncore);
565 struct intel_gt *gt = to_gt(i915);
566 u32 master_ctl;
567 u32 gu_misc_iir;
568
569 if (!intel_irqs_enabled(i915))
570 return IRQ_NONE;
571
572 master_ctl = gen11_master_intr_disable(regs);
573 if (!master_ctl) {
574 gen11_master_intr_enable(regs);
575 return IRQ_NONE;
576 }
577
578 /* Find, queue (onto bottom-halves), then clear each source */
579 gen11_gt_irq_handler(gt, master_ctl);
580
581 /* IRQs are synced during runtime_suspend, we don't require a wakeref */
582 if (master_ctl & GEN11_DISPLAY_IRQ)
583 gen11_display_irq_handler(i915);
584
585 gu_misc_iir = gen11_gu_misc_irq_ack(i915, master_ctl);
586
587 gen11_master_intr_enable(regs);
588
589 gen11_gu_misc_irq_handler(i915, gu_misc_iir);
590
591 pmu_irq_stats(i915, IRQ_HANDLED);
592
593 return IRQ_HANDLED;
594 }
595
dg1_master_intr_disable(void __iomem * const regs)596 static inline u32 dg1_master_intr_disable(void __iomem * const regs)
597 {
598 u32 val;
599
600 /* First disable interrupts */
601 raw_reg_write(regs, DG1_MSTR_TILE_INTR, 0);
602
603 /* Get the indication levels and ack the master unit */
604 val = raw_reg_read(regs, DG1_MSTR_TILE_INTR);
605 if (unlikely(!val))
606 return 0;
607
608 raw_reg_write(regs, DG1_MSTR_TILE_INTR, val);
609
610 return val;
611 }
612
dg1_master_intr_enable(void __iomem * const regs)613 static inline void dg1_master_intr_enable(void __iomem * const regs)
614 {
615 raw_reg_write(regs, DG1_MSTR_TILE_INTR, DG1_MSTR_IRQ);
616 }
617
dg1_irq_handler(int irq,void * arg)618 static irqreturn_t dg1_irq_handler(int irq, void *arg)
619 {
620 struct drm_i915_private * const i915 = arg;
621 struct intel_gt *gt = to_gt(i915);
622 void __iomem * const regs = intel_uncore_regs(gt->uncore);
623 u32 master_tile_ctl, master_ctl;
624 u32 gu_misc_iir;
625
626 if (!intel_irqs_enabled(i915))
627 return IRQ_NONE;
628
629 master_tile_ctl = dg1_master_intr_disable(regs);
630 if (!master_tile_ctl) {
631 dg1_master_intr_enable(regs);
632 return IRQ_NONE;
633 }
634
635 /* FIXME: we only support tile 0 for now. */
636 if (master_tile_ctl & DG1_MSTR_TILE(0)) {
637 master_ctl = raw_reg_read(regs, GEN11_GFX_MSTR_IRQ);
638 raw_reg_write(regs, GEN11_GFX_MSTR_IRQ, master_ctl);
639 } else {
640 drm_err(&i915->drm, "Tile not supported: 0x%08x\n",
641 master_tile_ctl);
642 dg1_master_intr_enable(regs);
643 return IRQ_NONE;
644 }
645
646 gen11_gt_irq_handler(gt, master_ctl);
647
648 if (master_ctl & GEN11_DISPLAY_IRQ)
649 gen11_display_irq_handler(i915);
650
651 gu_misc_iir = gen11_gu_misc_irq_ack(i915, master_ctl);
652
653 dg1_master_intr_enable(regs);
654
655 gen11_gu_misc_irq_handler(i915, gu_misc_iir);
656
657 pmu_irq_stats(i915, IRQ_HANDLED);
658
659 return IRQ_HANDLED;
660 }
661
ibx_irq_reset(struct drm_i915_private * dev_priv)662 static void ibx_irq_reset(struct drm_i915_private *dev_priv)
663 {
664 struct intel_uncore *uncore = &dev_priv->uncore;
665
666 if (HAS_PCH_NOP(dev_priv))
667 return;
668
669 GEN3_IRQ_RESET(uncore, SDE);
670
671 if (HAS_PCH_CPT(dev_priv) || HAS_PCH_LPT(dev_priv))
672 intel_uncore_write(&dev_priv->uncore, SERR_INT, 0xffffffff);
673 }
674
675 /* drm_dma.h hooks
676 */
ilk_irq_reset(struct drm_i915_private * dev_priv)677 static void ilk_irq_reset(struct drm_i915_private *dev_priv)
678 {
679 struct intel_uncore *uncore = &dev_priv->uncore;
680
681 GEN3_IRQ_RESET(uncore, DE);
682 dev_priv->irq_mask = ~0u;
683
684 if (GRAPHICS_VER(dev_priv) == 7)
685 intel_uncore_write(uncore, GEN7_ERR_INT, 0xffffffff);
686
687 if (IS_HASWELL(dev_priv)) {
688 intel_uncore_write(uncore, EDP_PSR_IMR, 0xffffffff);
689 intel_uncore_write(uncore, EDP_PSR_IIR, 0xffffffff);
690 }
691
692 gen5_gt_irq_reset(to_gt(dev_priv));
693
694 ibx_irq_reset(dev_priv);
695 }
696
valleyview_irq_reset(struct drm_i915_private * dev_priv)697 static void valleyview_irq_reset(struct drm_i915_private *dev_priv)
698 {
699 intel_uncore_write(&dev_priv->uncore, VLV_MASTER_IER, 0);
700 intel_uncore_posting_read(&dev_priv->uncore, VLV_MASTER_IER);
701
702 gen5_gt_irq_reset(to_gt(dev_priv));
703
704 spin_lock_irq(&dev_priv->irq_lock);
705 if (dev_priv->display_irqs_enabled)
706 vlv_display_irq_reset(dev_priv);
707 spin_unlock_irq(&dev_priv->irq_lock);
708 }
709
gen8_irq_reset(struct drm_i915_private * dev_priv)710 static void gen8_irq_reset(struct drm_i915_private *dev_priv)
711 {
712 struct intel_uncore *uncore = &dev_priv->uncore;
713
714 gen8_master_intr_disable(intel_uncore_regs(uncore));
715
716 gen8_gt_irq_reset(to_gt(dev_priv));
717 gen8_display_irq_reset(dev_priv);
718 GEN3_IRQ_RESET(uncore, GEN8_PCU_);
719
720 if (HAS_PCH_SPLIT(dev_priv))
721 ibx_irq_reset(dev_priv);
722
723 }
724
gen11_irq_reset(struct drm_i915_private * dev_priv)725 static void gen11_irq_reset(struct drm_i915_private *dev_priv)
726 {
727 struct intel_gt *gt = to_gt(dev_priv);
728 struct intel_uncore *uncore = gt->uncore;
729
730 gen11_master_intr_disable(intel_uncore_regs(&dev_priv->uncore));
731
732 gen11_gt_irq_reset(gt);
733 gen11_display_irq_reset(dev_priv);
734
735 GEN3_IRQ_RESET(uncore, GEN11_GU_MISC_);
736 GEN3_IRQ_RESET(uncore, GEN8_PCU_);
737 }
738
dg1_irq_reset(struct drm_i915_private * dev_priv)739 static void dg1_irq_reset(struct drm_i915_private *dev_priv)
740 {
741 struct intel_uncore *uncore = &dev_priv->uncore;
742 struct intel_gt *gt;
743 unsigned int i;
744
745 dg1_master_intr_disable(intel_uncore_regs(&dev_priv->uncore));
746
747 for_each_gt(gt, dev_priv, i)
748 gen11_gt_irq_reset(gt);
749
750 gen11_display_irq_reset(dev_priv);
751
752 GEN3_IRQ_RESET(uncore, GEN11_GU_MISC_);
753 GEN3_IRQ_RESET(uncore, GEN8_PCU_);
754 }
755
cherryview_irq_reset(struct drm_i915_private * dev_priv)756 static void cherryview_irq_reset(struct drm_i915_private *dev_priv)
757 {
758 struct intel_uncore *uncore = &dev_priv->uncore;
759
760 intel_uncore_write(uncore, GEN8_MASTER_IRQ, 0);
761 intel_uncore_posting_read(&dev_priv->uncore, GEN8_MASTER_IRQ);
762
763 gen8_gt_irq_reset(to_gt(dev_priv));
764
765 GEN3_IRQ_RESET(uncore, GEN8_PCU_);
766
767 spin_lock_irq(&dev_priv->irq_lock);
768 if (dev_priv->display_irqs_enabled)
769 vlv_display_irq_reset(dev_priv);
770 spin_unlock_irq(&dev_priv->irq_lock);
771 }
772
ilk_irq_postinstall(struct drm_i915_private * dev_priv)773 static void ilk_irq_postinstall(struct drm_i915_private *dev_priv)
774 {
775 gen5_gt_irq_postinstall(to_gt(dev_priv));
776
777 ilk_de_irq_postinstall(dev_priv);
778 }
779
valleyview_irq_postinstall(struct drm_i915_private * dev_priv)780 static void valleyview_irq_postinstall(struct drm_i915_private *dev_priv)
781 {
782 gen5_gt_irq_postinstall(to_gt(dev_priv));
783
784 spin_lock_irq(&dev_priv->irq_lock);
785 if (dev_priv->display_irqs_enabled)
786 vlv_display_irq_postinstall(dev_priv);
787 spin_unlock_irq(&dev_priv->irq_lock);
788
789 intel_uncore_write(&dev_priv->uncore, VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE);
790 intel_uncore_posting_read(&dev_priv->uncore, VLV_MASTER_IER);
791 }
792
gen8_irq_postinstall(struct drm_i915_private * dev_priv)793 static void gen8_irq_postinstall(struct drm_i915_private *dev_priv)
794 {
795 gen8_gt_irq_postinstall(to_gt(dev_priv));
796 gen8_de_irq_postinstall(dev_priv);
797
798 gen8_master_intr_enable(intel_uncore_regs(&dev_priv->uncore));
799 }
800
gen11_irq_postinstall(struct drm_i915_private * dev_priv)801 static void gen11_irq_postinstall(struct drm_i915_private *dev_priv)
802 {
803 struct intel_gt *gt = to_gt(dev_priv);
804 struct intel_uncore *uncore = gt->uncore;
805 u32 gu_misc_masked = GEN11_GU_MISC_GSE;
806
807 gen11_gt_irq_postinstall(gt);
808 gen11_de_irq_postinstall(dev_priv);
809
810 GEN3_IRQ_INIT(uncore, GEN11_GU_MISC_, ~gu_misc_masked, gu_misc_masked);
811
812 gen11_master_intr_enable(intel_uncore_regs(uncore));
813 intel_uncore_posting_read(&dev_priv->uncore, GEN11_GFX_MSTR_IRQ);
814 }
815
dg1_irq_postinstall(struct drm_i915_private * dev_priv)816 static void dg1_irq_postinstall(struct drm_i915_private *dev_priv)
817 {
818 struct intel_uncore *uncore = &dev_priv->uncore;
819 u32 gu_misc_masked = GEN11_GU_MISC_GSE;
820 struct intel_gt *gt;
821 unsigned int i;
822
823 for_each_gt(gt, dev_priv, i)
824 gen11_gt_irq_postinstall(gt);
825
826 GEN3_IRQ_INIT(uncore, GEN11_GU_MISC_, ~gu_misc_masked, gu_misc_masked);
827
828 dg1_de_irq_postinstall(dev_priv);
829
830 dg1_master_intr_enable(intel_uncore_regs(uncore));
831 intel_uncore_posting_read(uncore, DG1_MSTR_TILE_INTR);
832 }
833
cherryview_irq_postinstall(struct drm_i915_private * dev_priv)834 static void cherryview_irq_postinstall(struct drm_i915_private *dev_priv)
835 {
836 gen8_gt_irq_postinstall(to_gt(dev_priv));
837
838 spin_lock_irq(&dev_priv->irq_lock);
839 if (dev_priv->display_irqs_enabled)
840 vlv_display_irq_postinstall(dev_priv);
841 spin_unlock_irq(&dev_priv->irq_lock);
842
843 intel_uncore_write(&dev_priv->uncore, GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
844 intel_uncore_posting_read(&dev_priv->uncore, GEN8_MASTER_IRQ);
845 }
846
i8xx_irq_reset(struct drm_i915_private * dev_priv)847 static void i8xx_irq_reset(struct drm_i915_private *dev_priv)
848 {
849 struct intel_uncore *uncore = &dev_priv->uncore;
850
851 i9xx_pipestat_irq_reset(dev_priv);
852
853 gen2_irq_reset(uncore);
854 dev_priv->irq_mask = ~0u;
855 }
856
i9xx_error_mask(struct drm_i915_private * i915)857 static u32 i9xx_error_mask(struct drm_i915_private *i915)
858 {
859 /*
860 * On gen2/3 FBC generates (seemingly spurious)
861 * display INVALID_GTT/INVALID_GTT_PTE table errors.
862 *
863 * Also gen3 bspec has this to say:
864 * "DISPA_INVALID_GTT_PTE
865 " [DevNapa] : Reserved. This bit does not reflect the page
866 " table error for the display plane A."
867 *
868 * Unfortunately we can't mask off individual PGTBL_ER bits,
869 * so we just have to mask off all page table errors via EMR.
870 */
871 if (HAS_FBC(i915))
872 return ~I915_ERROR_MEMORY_REFRESH;
873 else
874 return ~(I915_ERROR_PAGE_TABLE |
875 I915_ERROR_MEMORY_REFRESH);
876 }
877
i8xx_irq_postinstall(struct drm_i915_private * dev_priv)878 static void i8xx_irq_postinstall(struct drm_i915_private *dev_priv)
879 {
880 struct intel_uncore *uncore = &dev_priv->uncore;
881 u16 enable_mask;
882
883 intel_uncore_write16(uncore, EMR, i9xx_error_mask(dev_priv));
884
885 /* Unmask the interrupts that we always want on. */
886 dev_priv->irq_mask =
887 ~(I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
888 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
889 I915_MASTER_ERROR_INTERRUPT);
890
891 enable_mask =
892 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
893 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
894 I915_MASTER_ERROR_INTERRUPT |
895 I915_USER_INTERRUPT;
896
897 gen2_irq_init(uncore, dev_priv->irq_mask, enable_mask);
898
899 /* Interrupt setup is already guaranteed to be single-threaded, this is
900 * just to make the assert_spin_locked check happy. */
901 spin_lock_irq(&dev_priv->irq_lock);
902 i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
903 i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
904 spin_unlock_irq(&dev_priv->irq_lock);
905 }
906
i8xx_error_irq_ack(struct drm_i915_private * i915,u16 * eir,u16 * eir_stuck)907 static void i8xx_error_irq_ack(struct drm_i915_private *i915,
908 u16 *eir, u16 *eir_stuck)
909 {
910 struct intel_uncore *uncore = &i915->uncore;
911 u16 emr;
912
913 *eir = intel_uncore_read16(uncore, EIR);
914 intel_uncore_write16(uncore, EIR, *eir);
915
916 *eir_stuck = intel_uncore_read16(uncore, EIR);
917 if (*eir_stuck == 0)
918 return;
919
920 /*
921 * Toggle all EMR bits to make sure we get an edge
922 * in the ISR master error bit if we don't clear
923 * all the EIR bits. Otherwise the edge triggered
924 * IIR on i965/g4x wouldn't notice that an interrupt
925 * is still pending. Also some EIR bits can't be
926 * cleared except by handling the underlying error
927 * (or by a GPU reset) so we mask any bit that
928 * remains set.
929 */
930 emr = intel_uncore_read16(uncore, EMR);
931 intel_uncore_write16(uncore, EMR, 0xffff);
932 intel_uncore_write16(uncore, EMR, emr | *eir_stuck);
933 }
934
i8xx_error_irq_handler(struct drm_i915_private * dev_priv,u16 eir,u16 eir_stuck)935 static void i8xx_error_irq_handler(struct drm_i915_private *dev_priv,
936 u16 eir, u16 eir_stuck)
937 {
938 drm_dbg(&dev_priv->drm, "Master Error: EIR 0x%04x\n", eir);
939
940 if (eir_stuck)
941 drm_dbg(&dev_priv->drm, "EIR stuck: 0x%04x, masked\n",
942 eir_stuck);
943
944 drm_dbg(&dev_priv->drm, "PGTBL_ER: 0x%08x\n",
945 intel_uncore_read(&dev_priv->uncore, PGTBL_ER));
946 }
947
i9xx_error_irq_ack(struct drm_i915_private * dev_priv,u32 * eir,u32 * eir_stuck)948 static void i9xx_error_irq_ack(struct drm_i915_private *dev_priv,
949 u32 *eir, u32 *eir_stuck)
950 {
951 u32 emr;
952
953 *eir = intel_uncore_read(&dev_priv->uncore, EIR);
954 intel_uncore_write(&dev_priv->uncore, EIR, *eir);
955
956 *eir_stuck = intel_uncore_read(&dev_priv->uncore, EIR);
957 if (*eir_stuck == 0)
958 return;
959
960 /*
961 * Toggle all EMR bits to make sure we get an edge
962 * in the ISR master error bit if we don't clear
963 * all the EIR bits. Otherwise the edge triggered
964 * IIR on i965/g4x wouldn't notice that an interrupt
965 * is still pending. Also some EIR bits can't be
966 * cleared except by handling the underlying error
967 * (or by a GPU reset) so we mask any bit that
968 * remains set.
969 */
970 emr = intel_uncore_read(&dev_priv->uncore, EMR);
971 intel_uncore_write(&dev_priv->uncore, EMR, 0xffffffff);
972 intel_uncore_write(&dev_priv->uncore, EMR, emr | *eir_stuck);
973 }
974
i9xx_error_irq_handler(struct drm_i915_private * dev_priv,u32 eir,u32 eir_stuck)975 static void i9xx_error_irq_handler(struct drm_i915_private *dev_priv,
976 u32 eir, u32 eir_stuck)
977 {
978 drm_dbg(&dev_priv->drm, "Master Error, EIR 0x%08x\n", eir);
979
980 if (eir_stuck)
981 drm_dbg(&dev_priv->drm, "EIR stuck: 0x%08x, masked\n",
982 eir_stuck);
983
984 drm_dbg(&dev_priv->drm, "PGTBL_ER: 0x%08x\n",
985 intel_uncore_read(&dev_priv->uncore, PGTBL_ER));
986 }
987
i8xx_irq_handler(int irq,void * arg)988 static irqreturn_t i8xx_irq_handler(int irq, void *arg)
989 {
990 struct drm_i915_private *dev_priv = arg;
991 irqreturn_t ret = IRQ_NONE;
992
993 if (!intel_irqs_enabled(dev_priv))
994 return IRQ_NONE;
995
996 /* IRQs are synced during runtime_suspend, we don't require a wakeref */
997 disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
998
999 do {
1000 u32 pipe_stats[I915_MAX_PIPES] = {};
1001 u16 eir = 0, eir_stuck = 0;
1002 u16 iir;
1003
1004 iir = intel_uncore_read16(&dev_priv->uncore, GEN2_IIR);
1005 if (iir == 0)
1006 break;
1007
1008 ret = IRQ_HANDLED;
1009
1010 /* Call regardless, as some status bits might not be
1011 * signalled in iir */
1012 i9xx_pipestat_irq_ack(dev_priv, iir, pipe_stats);
1013
1014 if (iir & I915_MASTER_ERROR_INTERRUPT)
1015 i8xx_error_irq_ack(dev_priv, &eir, &eir_stuck);
1016
1017 intel_uncore_write16(&dev_priv->uncore, GEN2_IIR, iir);
1018
1019 if (iir & I915_USER_INTERRUPT)
1020 intel_engine_cs_irq(to_gt(dev_priv)->engine[RCS0], iir);
1021
1022 if (iir & I915_MASTER_ERROR_INTERRUPT)
1023 i8xx_error_irq_handler(dev_priv, eir, eir_stuck);
1024
1025 i8xx_pipestat_irq_handler(dev_priv, iir, pipe_stats);
1026 } while (0);
1027
1028 pmu_irq_stats(dev_priv, ret);
1029
1030 enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
1031
1032 return ret;
1033 }
1034
i915_irq_reset(struct drm_i915_private * dev_priv)1035 static void i915_irq_reset(struct drm_i915_private *dev_priv)
1036 {
1037 struct intel_uncore *uncore = &dev_priv->uncore;
1038
1039 if (I915_HAS_HOTPLUG(dev_priv)) {
1040 i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
1041 intel_uncore_rmw(&dev_priv->uncore, PORT_HOTPLUG_STAT, 0, 0);
1042 }
1043
1044 i9xx_pipestat_irq_reset(dev_priv);
1045
1046 GEN3_IRQ_RESET(uncore, GEN2_);
1047 dev_priv->irq_mask = ~0u;
1048 }
1049
i915_irq_postinstall(struct drm_i915_private * dev_priv)1050 static void i915_irq_postinstall(struct drm_i915_private *dev_priv)
1051 {
1052 struct intel_uncore *uncore = &dev_priv->uncore;
1053 u32 enable_mask;
1054
1055 intel_uncore_write(uncore, EMR, i9xx_error_mask(dev_priv));
1056
1057 /* Unmask the interrupts that we always want on. */
1058 dev_priv->irq_mask =
1059 ~(I915_ASLE_INTERRUPT |
1060 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
1061 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
1062 I915_MASTER_ERROR_INTERRUPT);
1063
1064 enable_mask =
1065 I915_ASLE_INTERRUPT |
1066 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
1067 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
1068 I915_MASTER_ERROR_INTERRUPT |
1069 I915_USER_INTERRUPT;
1070
1071 if (I915_HAS_HOTPLUG(dev_priv)) {
1072 /* Enable in IER... */
1073 enable_mask |= I915_DISPLAY_PORT_INTERRUPT;
1074 /* and unmask in IMR */
1075 dev_priv->irq_mask &= ~I915_DISPLAY_PORT_INTERRUPT;
1076 }
1077
1078 GEN3_IRQ_INIT(uncore, GEN2_, dev_priv->irq_mask, enable_mask);
1079
1080 /* Interrupt setup is already guaranteed to be single-threaded, this is
1081 * just to make the assert_spin_locked check happy. */
1082 spin_lock_irq(&dev_priv->irq_lock);
1083 i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
1084 i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
1085 spin_unlock_irq(&dev_priv->irq_lock);
1086
1087 i915_enable_asle_pipestat(dev_priv);
1088 }
1089
i915_irq_handler(int irq,void * arg)1090 static irqreturn_t i915_irq_handler(int irq, void *arg)
1091 {
1092 struct drm_i915_private *dev_priv = arg;
1093 irqreturn_t ret = IRQ_NONE;
1094
1095 if (!intel_irqs_enabled(dev_priv))
1096 return IRQ_NONE;
1097
1098 /* IRQs are synced during runtime_suspend, we don't require a wakeref */
1099 disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
1100
1101 do {
1102 u32 pipe_stats[I915_MAX_PIPES] = {};
1103 u32 eir = 0, eir_stuck = 0;
1104 u32 hotplug_status = 0;
1105 u32 iir;
1106
1107 iir = intel_uncore_read(&dev_priv->uncore, GEN2_IIR);
1108 if (iir == 0)
1109 break;
1110
1111 ret = IRQ_HANDLED;
1112
1113 if (I915_HAS_HOTPLUG(dev_priv) &&
1114 iir & I915_DISPLAY_PORT_INTERRUPT)
1115 hotplug_status = i9xx_hpd_irq_ack(dev_priv);
1116
1117 /* Call regardless, as some status bits might not be
1118 * signalled in iir */
1119 i9xx_pipestat_irq_ack(dev_priv, iir, pipe_stats);
1120
1121 if (iir & I915_MASTER_ERROR_INTERRUPT)
1122 i9xx_error_irq_ack(dev_priv, &eir, &eir_stuck);
1123
1124 intel_uncore_write(&dev_priv->uncore, GEN2_IIR, iir);
1125
1126 if (iir & I915_USER_INTERRUPT)
1127 intel_engine_cs_irq(to_gt(dev_priv)->engine[RCS0], iir);
1128
1129 if (iir & I915_MASTER_ERROR_INTERRUPT)
1130 i9xx_error_irq_handler(dev_priv, eir, eir_stuck);
1131
1132 if (hotplug_status)
1133 i9xx_hpd_irq_handler(dev_priv, hotplug_status);
1134
1135 i915_pipestat_irq_handler(dev_priv, iir, pipe_stats);
1136 } while (0);
1137
1138 pmu_irq_stats(dev_priv, ret);
1139
1140 enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
1141
1142 return ret;
1143 }
1144
i965_irq_reset(struct drm_i915_private * dev_priv)1145 static void i965_irq_reset(struct drm_i915_private *dev_priv)
1146 {
1147 struct intel_uncore *uncore = &dev_priv->uncore;
1148
1149 i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
1150 intel_uncore_rmw(uncore, PORT_HOTPLUG_STAT, 0, 0);
1151
1152 i9xx_pipestat_irq_reset(dev_priv);
1153
1154 GEN3_IRQ_RESET(uncore, GEN2_);
1155 dev_priv->irq_mask = ~0u;
1156 }
1157
i965_error_mask(struct drm_i915_private * i915)1158 static u32 i965_error_mask(struct drm_i915_private *i915)
1159 {
1160 /*
1161 * Enable some error detection, note the instruction error mask
1162 * bit is reserved, so we leave it masked.
1163 *
1164 * i965 FBC no longer generates spurious GTT errors,
1165 * so we can always enable the page table errors.
1166 */
1167 if (IS_G4X(i915))
1168 return ~(GM45_ERROR_PAGE_TABLE |
1169 GM45_ERROR_MEM_PRIV |
1170 GM45_ERROR_CP_PRIV |
1171 I915_ERROR_MEMORY_REFRESH);
1172 else
1173 return ~(I915_ERROR_PAGE_TABLE |
1174 I915_ERROR_MEMORY_REFRESH);
1175 }
1176
i965_irq_postinstall(struct drm_i915_private * dev_priv)1177 static void i965_irq_postinstall(struct drm_i915_private *dev_priv)
1178 {
1179 struct intel_uncore *uncore = &dev_priv->uncore;
1180 u32 enable_mask;
1181
1182 intel_uncore_write(uncore, EMR, i965_error_mask(dev_priv));
1183
1184 /* Unmask the interrupts that we always want on. */
1185 dev_priv->irq_mask =
1186 ~(I915_ASLE_INTERRUPT |
1187 I915_DISPLAY_PORT_INTERRUPT |
1188 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
1189 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
1190 I915_MASTER_ERROR_INTERRUPT);
1191
1192 enable_mask =
1193 I915_ASLE_INTERRUPT |
1194 I915_DISPLAY_PORT_INTERRUPT |
1195 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
1196 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
1197 I915_MASTER_ERROR_INTERRUPT |
1198 I915_USER_INTERRUPT;
1199
1200 if (IS_G4X(dev_priv))
1201 enable_mask |= I915_BSD_USER_INTERRUPT;
1202
1203 GEN3_IRQ_INIT(uncore, GEN2_, dev_priv->irq_mask, enable_mask);
1204
1205 /* Interrupt setup is already guaranteed to be single-threaded, this is
1206 * just to make the assert_spin_locked check happy. */
1207 spin_lock_irq(&dev_priv->irq_lock);
1208 i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
1209 i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
1210 i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
1211 spin_unlock_irq(&dev_priv->irq_lock);
1212
1213 i915_enable_asle_pipestat(dev_priv);
1214 }
1215
i965_irq_handler(int irq,void * arg)1216 static irqreturn_t i965_irq_handler(int irq, void *arg)
1217 {
1218 struct drm_i915_private *dev_priv = arg;
1219 irqreturn_t ret = IRQ_NONE;
1220
1221 if (!intel_irqs_enabled(dev_priv))
1222 return IRQ_NONE;
1223
1224 /* IRQs are synced during runtime_suspend, we don't require a wakeref */
1225 disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
1226
1227 do {
1228 u32 pipe_stats[I915_MAX_PIPES] = {};
1229 u32 eir = 0, eir_stuck = 0;
1230 u32 hotplug_status = 0;
1231 u32 iir;
1232
1233 iir = intel_uncore_read(&dev_priv->uncore, GEN2_IIR);
1234 if (iir == 0)
1235 break;
1236
1237 ret = IRQ_HANDLED;
1238
1239 if (iir & I915_DISPLAY_PORT_INTERRUPT)
1240 hotplug_status = i9xx_hpd_irq_ack(dev_priv);
1241
1242 /* Call regardless, as some status bits might not be
1243 * signalled in iir */
1244 i9xx_pipestat_irq_ack(dev_priv, iir, pipe_stats);
1245
1246 if (iir & I915_MASTER_ERROR_INTERRUPT)
1247 i9xx_error_irq_ack(dev_priv, &eir, &eir_stuck);
1248
1249 intel_uncore_write(&dev_priv->uncore, GEN2_IIR, iir);
1250
1251 if (iir & I915_USER_INTERRUPT)
1252 intel_engine_cs_irq(to_gt(dev_priv)->engine[RCS0],
1253 iir);
1254
1255 if (iir & I915_BSD_USER_INTERRUPT)
1256 intel_engine_cs_irq(to_gt(dev_priv)->engine[VCS0],
1257 iir >> 25);
1258
1259 if (iir & I915_MASTER_ERROR_INTERRUPT)
1260 i9xx_error_irq_handler(dev_priv, eir, eir_stuck);
1261
1262 if (hotplug_status)
1263 i9xx_hpd_irq_handler(dev_priv, hotplug_status);
1264
1265 i965_pipestat_irq_handler(dev_priv, iir, pipe_stats);
1266 } while (0);
1267
1268 pmu_irq_stats(dev_priv, IRQ_HANDLED);
1269
1270 enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
1271
1272 return ret;
1273 }
1274
1275 /**
1276 * intel_irq_init - initializes irq support
1277 * @dev_priv: i915 device instance
1278 *
1279 * This function initializes all the irq support including work items, timers
1280 * and all the vtables. It does not setup the interrupt itself though.
1281 */
intel_irq_init(struct drm_i915_private * dev_priv)1282 void intel_irq_init(struct drm_i915_private *dev_priv)
1283 {
1284 int i;
1285
1286 INIT_WORK(&dev_priv->l3_parity.error_work, ivb_parity_work);
1287 for (i = 0; i < MAX_L3_SLICES; ++i)
1288 dev_priv->l3_parity.remap_info[i] = NULL;
1289
1290 /* pre-gen11 the guc irqs bits are in the upper 16 bits of the pm reg */
1291 if (HAS_GT_UC(dev_priv) && GRAPHICS_VER(dev_priv) < 11)
1292 to_gt(dev_priv)->pm_guc_events = GUC_INTR_GUC2HOST << 16;
1293 }
1294
1295 /**
1296 * intel_irq_fini - deinitializes IRQ support
1297 * @i915: i915 device instance
1298 *
1299 * This function deinitializes all the IRQ support.
1300 */
intel_irq_fini(struct drm_i915_private * i915)1301 void intel_irq_fini(struct drm_i915_private *i915)
1302 {
1303 int i;
1304
1305 for (i = 0; i < MAX_L3_SLICES; ++i)
1306 kfree(i915->l3_parity.remap_info[i]);
1307 }
1308
intel_irq_handler(struct drm_i915_private * dev_priv)1309 static irq_handler_t intel_irq_handler(struct drm_i915_private *dev_priv)
1310 {
1311 if (HAS_GMCH(dev_priv)) {
1312 if (IS_CHERRYVIEW(dev_priv))
1313 return cherryview_irq_handler;
1314 else if (IS_VALLEYVIEW(dev_priv))
1315 return valleyview_irq_handler;
1316 else if (GRAPHICS_VER(dev_priv) == 4)
1317 return i965_irq_handler;
1318 else if (GRAPHICS_VER(dev_priv) == 3)
1319 return i915_irq_handler;
1320 else
1321 return i8xx_irq_handler;
1322 } else {
1323 if (GRAPHICS_VER_FULL(dev_priv) >= IP_VER(12, 10))
1324 return dg1_irq_handler;
1325 else if (GRAPHICS_VER(dev_priv) >= 11)
1326 return gen11_irq_handler;
1327 else if (GRAPHICS_VER(dev_priv) >= 8)
1328 return gen8_irq_handler;
1329 else
1330 return ilk_irq_handler;
1331 }
1332 }
1333
intel_irq_reset(struct drm_i915_private * dev_priv)1334 static void intel_irq_reset(struct drm_i915_private *dev_priv)
1335 {
1336 if (HAS_GMCH(dev_priv)) {
1337 if (IS_CHERRYVIEW(dev_priv))
1338 cherryview_irq_reset(dev_priv);
1339 else if (IS_VALLEYVIEW(dev_priv))
1340 valleyview_irq_reset(dev_priv);
1341 else if (GRAPHICS_VER(dev_priv) == 4)
1342 i965_irq_reset(dev_priv);
1343 else if (GRAPHICS_VER(dev_priv) == 3)
1344 i915_irq_reset(dev_priv);
1345 else
1346 i8xx_irq_reset(dev_priv);
1347 } else {
1348 if (GRAPHICS_VER_FULL(dev_priv) >= IP_VER(12, 10))
1349 dg1_irq_reset(dev_priv);
1350 else if (GRAPHICS_VER(dev_priv) >= 11)
1351 gen11_irq_reset(dev_priv);
1352 else if (GRAPHICS_VER(dev_priv) >= 8)
1353 gen8_irq_reset(dev_priv);
1354 else
1355 ilk_irq_reset(dev_priv);
1356 }
1357 }
1358
intel_irq_postinstall(struct drm_i915_private * dev_priv)1359 static void intel_irq_postinstall(struct drm_i915_private *dev_priv)
1360 {
1361 if (HAS_GMCH(dev_priv)) {
1362 if (IS_CHERRYVIEW(dev_priv))
1363 cherryview_irq_postinstall(dev_priv);
1364 else if (IS_VALLEYVIEW(dev_priv))
1365 valleyview_irq_postinstall(dev_priv);
1366 else if (GRAPHICS_VER(dev_priv) == 4)
1367 i965_irq_postinstall(dev_priv);
1368 else if (GRAPHICS_VER(dev_priv) == 3)
1369 i915_irq_postinstall(dev_priv);
1370 else
1371 i8xx_irq_postinstall(dev_priv);
1372 } else {
1373 if (GRAPHICS_VER_FULL(dev_priv) >= IP_VER(12, 10))
1374 dg1_irq_postinstall(dev_priv);
1375 else if (GRAPHICS_VER(dev_priv) >= 11)
1376 gen11_irq_postinstall(dev_priv);
1377 else if (GRAPHICS_VER(dev_priv) >= 8)
1378 gen8_irq_postinstall(dev_priv);
1379 else
1380 ilk_irq_postinstall(dev_priv);
1381 }
1382 }
1383
1384 /**
1385 * intel_irq_install - enables the hardware interrupt
1386 * @dev_priv: i915 device instance
1387 *
1388 * This function enables the hardware interrupt handling, but leaves the hotplug
1389 * handling still disabled. It is called after intel_irq_init().
1390 *
1391 * In the driver load and resume code we need working interrupts in a few places
1392 * but don't want to deal with the hassle of concurrent probe and hotplug
1393 * workers. Hence the split into this two-stage approach.
1394 */
intel_irq_install(struct drm_i915_private * dev_priv)1395 int intel_irq_install(struct drm_i915_private *dev_priv)
1396 {
1397 int irq = to_pci_dev(dev_priv->drm.dev)->irq;
1398 int ret;
1399
1400 /*
1401 * We enable some interrupt sources in our postinstall hooks, so mark
1402 * interrupts as enabled _before_ actually enabling them to avoid
1403 * special cases in our ordering checks.
1404 */
1405 dev_priv->runtime_pm.irqs_enabled = true;
1406
1407 dev_priv->irq_enabled = true;
1408
1409 intel_irq_reset(dev_priv);
1410
1411 ret = request_irq(irq, intel_irq_handler(dev_priv),
1412 IRQF_SHARED, DRIVER_NAME, dev_priv);
1413 if (ret < 0) {
1414 dev_priv->irq_enabled = false;
1415 return ret;
1416 }
1417
1418 intel_irq_postinstall(dev_priv);
1419
1420 return ret;
1421 }
1422
1423 /**
1424 * intel_irq_uninstall - finilizes all irq handling
1425 * @dev_priv: i915 device instance
1426 *
1427 * This stops interrupt and hotplug handling and unregisters and frees all
1428 * resources acquired in the init functions.
1429 */
intel_irq_uninstall(struct drm_i915_private * dev_priv)1430 void intel_irq_uninstall(struct drm_i915_private *dev_priv)
1431 {
1432 int irq = to_pci_dev(dev_priv->drm.dev)->irq;
1433
1434 /*
1435 * FIXME we can get called twice during driver probe
1436 * error handling as well as during driver remove due to
1437 * intel_display_driver_remove() calling us out of sequence.
1438 * Would be nice if it didn't do that...
1439 */
1440 if (!dev_priv->irq_enabled)
1441 return;
1442
1443 dev_priv->irq_enabled = false;
1444
1445 intel_irq_reset(dev_priv);
1446
1447 free_irq(irq, dev_priv);
1448
1449 intel_hpd_cancel_work(dev_priv);
1450 dev_priv->runtime_pm.irqs_enabled = false;
1451 }
1452
1453 /**
1454 * intel_runtime_pm_disable_interrupts - runtime interrupt disabling
1455 * @dev_priv: i915 device instance
1456 *
1457 * This function is used to disable interrupts at runtime, both in the runtime
1458 * pm and the system suspend/resume code.
1459 */
intel_runtime_pm_disable_interrupts(struct drm_i915_private * dev_priv)1460 void intel_runtime_pm_disable_interrupts(struct drm_i915_private *dev_priv)
1461 {
1462 intel_irq_reset(dev_priv);
1463 dev_priv->runtime_pm.irqs_enabled = false;
1464 intel_synchronize_irq(dev_priv);
1465 }
1466
1467 /**
1468 * intel_runtime_pm_enable_interrupts - runtime interrupt enabling
1469 * @dev_priv: i915 device instance
1470 *
1471 * This function is used to enable interrupts at runtime, both in the runtime
1472 * pm and the system suspend/resume code.
1473 */
intel_runtime_pm_enable_interrupts(struct drm_i915_private * dev_priv)1474 void intel_runtime_pm_enable_interrupts(struct drm_i915_private *dev_priv)
1475 {
1476 dev_priv->runtime_pm.irqs_enabled = true;
1477 intel_irq_reset(dev_priv);
1478 intel_irq_postinstall(dev_priv);
1479 }
1480
intel_irqs_enabled(struct drm_i915_private * dev_priv)1481 bool intel_irqs_enabled(struct drm_i915_private *dev_priv)
1482 {
1483 return dev_priv->runtime_pm.irqs_enabled;
1484 }
1485
intel_synchronize_irq(struct drm_i915_private * i915)1486 void intel_synchronize_irq(struct drm_i915_private *i915)
1487 {
1488 synchronize_irq(to_pci_dev(i915->drm.dev)->irq);
1489 }
1490
intel_synchronize_hardirq(struct drm_i915_private * i915)1491 void intel_synchronize_hardirq(struct drm_i915_private *i915)
1492 {
1493 synchronize_hardirq(to_pci_dev(i915->drm.dev)->irq);
1494 }
1495