1 // SPDX-License-Identifier: MIT
3 * Copyright © 2019 Intel Corporation
6 #include <linux/sched/clock.h>
10 #include "intel_breadcrumbs.h"
12 #include "intel_gt_irq.h"
13 #include "intel_gt_regs.h"
14 #include "intel_uncore.h"
15 #include "intel_rps.h"
16 #include "pxp/intel_pxp_irq.h"
18 static void guc_irq_handler(struct intel_guc *guc, u16 iir)
20 if (iir & GUC_INTR_GUC2HOST)
21 intel_guc_to_host_event_handler(guc);
25 gen11_gt_engine_identity(struct intel_gt *gt,
26 const unsigned int bank, const unsigned int bit)
28 void __iomem * const regs = gt->uncore->regs;
32 lockdep_assert_held(>->irq_lock);
34 raw_reg_write(regs, GEN11_IIR_REG_SELECTOR(bank), BIT(bit));
37 * NB: Specs do not specify how long to spin wait,
38 * so we do ~100us as an educated guess.
40 timeout_ts = (local_clock() >> 10) + 100;
42 ident = raw_reg_read(regs, GEN11_INTR_IDENTITY_REG(bank));
43 } while (!(ident & GEN11_INTR_DATA_VALID) &&
44 !time_after32(local_clock() >> 10, timeout_ts));
46 if (unlikely(!(ident & GEN11_INTR_DATA_VALID))) {
47 DRM_ERROR("INTR_IDENTITY_REG%u:%u 0x%08x not valid!\n",
52 raw_reg_write(regs, GEN11_INTR_IDENTITY_REG(bank),
53 GEN11_INTR_DATA_VALID);
59 gen11_other_irq_handler(struct intel_gt *gt, const u8 instance,
62 if (instance == OTHER_GUC_INSTANCE)
63 return guc_irq_handler(>->uc.guc, iir);
65 if (instance == OTHER_GTPM_INSTANCE)
66 return gen11_rps_irq_handler(>->rps, iir);
68 if (instance == OTHER_KCR_INSTANCE)
69 return intel_pxp_irq_handler(>->pxp, iir);
71 if (instance == OTHER_GSC_INSTANCE)
72 return intel_gsc_irq_handler(gt, iir);
74 WARN_ONCE(1, "unhandled other interrupt instance=0x%x, iir=0x%x\n",
79 gen11_engine_irq_handler(struct intel_gt *gt, const u8 class,
80 const u8 instance, const u16 iir)
82 struct intel_engine_cs *engine;
84 if (instance <= MAX_ENGINE_INSTANCE)
85 engine = gt->engine_class[class][instance];
90 return intel_engine_cs_irq(engine, iir);
92 WARN_ONCE(1, "unhandled engine interrupt class=0x%x, instance=0x%x\n",
97 gen11_gt_identity_handler(struct intel_gt *gt, const u32 identity)
99 const u8 class = GEN11_INTR_ENGINE_CLASS(identity);
100 const u8 instance = GEN11_INTR_ENGINE_INSTANCE(identity);
101 const u16 intr = GEN11_INTR_ENGINE_INTR(identity);
106 if (class <= COPY_ENGINE_CLASS || class == COMPUTE_CLASS)
107 return gen11_engine_irq_handler(gt, class, instance, intr);
109 if (class == OTHER_CLASS)
110 return gen11_other_irq_handler(gt, instance, intr);
112 WARN_ONCE(1, "unknown interrupt class=0x%x, instance=0x%x, intr=0x%x\n",
113 class, instance, intr);
117 gen11_gt_bank_handler(struct intel_gt *gt, const unsigned int bank)
119 void __iomem * const regs = gt->uncore->regs;
120 unsigned long intr_dw;
123 lockdep_assert_held(>->irq_lock);
125 intr_dw = raw_reg_read(regs, GEN11_GT_INTR_DW(bank));
127 for_each_set_bit(bit, &intr_dw, 32) {
128 const u32 ident = gen11_gt_engine_identity(gt, bank, bit);
130 gen11_gt_identity_handler(gt, ident);
133 /* Clear must be after shared has been served for engine */
134 raw_reg_write(regs, GEN11_GT_INTR_DW(bank), intr_dw);
137 void gen11_gt_irq_handler(struct intel_gt *gt, const u32 master_ctl)
141 spin_lock(>->irq_lock);
143 for (bank = 0; bank < 2; bank++) {
144 if (master_ctl & GEN11_GT_DW_IRQ(bank))
145 gen11_gt_bank_handler(gt, bank);
148 spin_unlock(>->irq_lock);
151 bool gen11_gt_reset_one_iir(struct intel_gt *gt,
152 const unsigned int bank, const unsigned int bit)
154 void __iomem * const regs = gt->uncore->regs;
157 lockdep_assert_held(>->irq_lock);
159 dw = raw_reg_read(regs, GEN11_GT_INTR_DW(bank));
162 * According to the BSpec, DW_IIR bits cannot be cleared without
163 * first servicing the Selector & Shared IIR registers.
165 gen11_gt_engine_identity(gt, bank, bit);
168 * We locked GT INT DW by reading it. If we want to (try
169 * to) recover from this successfully, we need to clear
170 * our bit, otherwise we are locking the register for
173 raw_reg_write(regs, GEN11_GT_INTR_DW(bank), BIT(bit));
181 void gen11_gt_irq_reset(struct intel_gt *gt)
183 struct intel_uncore *uncore = gt->uncore;
185 /* Disable RCS, BCS, VCS and VECS class engines. */
186 intel_uncore_write(uncore, GEN11_RENDER_COPY_INTR_ENABLE, 0);
187 intel_uncore_write(uncore, GEN11_VCS_VECS_INTR_ENABLE, 0);
189 intel_uncore_write(uncore, GEN12_CCS_RSVD_INTR_ENABLE, 0);
190 if (HAS_HECI_GSC(gt->i915))
191 intel_uncore_write(uncore, GEN11_GUNIT_CSME_INTR_ENABLE, 0);
193 /* Restore masks irqs on RCS, BCS, VCS and VECS engines. */
194 intel_uncore_write(uncore, GEN11_RCS0_RSVD_INTR_MASK, ~0);
195 intel_uncore_write(uncore, GEN11_BCS_RSVD_INTR_MASK, ~0);
196 if (HAS_ENGINE(gt, BCS1) || HAS_ENGINE(gt, BCS2))
197 intel_uncore_write(uncore, XEHPC_BCS1_BCS2_INTR_MASK, ~0);
198 if (HAS_ENGINE(gt, BCS3) || HAS_ENGINE(gt, BCS4))
199 intel_uncore_write(uncore, XEHPC_BCS3_BCS4_INTR_MASK, ~0);
200 if (HAS_ENGINE(gt, BCS5) || HAS_ENGINE(gt, BCS6))
201 intel_uncore_write(uncore, XEHPC_BCS5_BCS6_INTR_MASK, ~0);
202 if (HAS_ENGINE(gt, BCS7) || HAS_ENGINE(gt, BCS8))
203 intel_uncore_write(uncore, XEHPC_BCS7_BCS8_INTR_MASK, ~0);
204 intel_uncore_write(uncore, GEN11_VCS0_VCS1_INTR_MASK, ~0);
205 intel_uncore_write(uncore, GEN11_VCS2_VCS3_INTR_MASK, ~0);
206 if (HAS_ENGINE(gt, VCS4) || HAS_ENGINE(gt, VCS5))
207 intel_uncore_write(uncore, GEN12_VCS4_VCS5_INTR_MASK, ~0);
208 if (HAS_ENGINE(gt, VCS6) || HAS_ENGINE(gt, VCS7))
209 intel_uncore_write(uncore, GEN12_VCS6_VCS7_INTR_MASK, ~0);
210 intel_uncore_write(uncore, GEN11_VECS0_VECS1_INTR_MASK, ~0);
211 if (HAS_ENGINE(gt, VECS2) || HAS_ENGINE(gt, VECS3))
212 intel_uncore_write(uncore, GEN12_VECS2_VECS3_INTR_MASK, ~0);
213 if (HAS_ENGINE(gt, CCS0) || HAS_ENGINE(gt, CCS1))
214 intel_uncore_write(uncore, GEN12_CCS0_CCS1_INTR_MASK, ~0);
215 if (HAS_ENGINE(gt, CCS2) || HAS_ENGINE(gt, CCS3))
216 intel_uncore_write(uncore, GEN12_CCS2_CCS3_INTR_MASK, ~0);
217 if (HAS_HECI_GSC(gt->i915))
218 intel_uncore_write(uncore, GEN11_GUNIT_CSME_INTR_MASK, ~0);
220 intel_uncore_write(uncore, GEN11_GPM_WGBOXPERF_INTR_ENABLE, 0);
221 intel_uncore_write(uncore, GEN11_GPM_WGBOXPERF_INTR_MASK, ~0);
222 intel_uncore_write(uncore, GEN11_GUC_SG_INTR_ENABLE, 0);
223 intel_uncore_write(uncore, GEN11_GUC_SG_INTR_MASK, ~0);
225 intel_uncore_write(uncore, GEN11_CRYPTO_RSVD_INTR_ENABLE, 0);
226 intel_uncore_write(uncore, GEN11_CRYPTO_RSVD_INTR_MASK, ~0);
229 void gen11_gt_irq_postinstall(struct intel_gt *gt)
231 struct intel_uncore *uncore = gt->uncore;
232 u32 irqs = GT_RENDER_USER_INTERRUPT;
233 const u32 gsc_mask = GSC_IRQ_INTF(0) | GSC_IRQ_INTF(1);
237 if (!intel_uc_wants_guc_submission(>->uc))
238 irqs |= GT_CS_MASTER_ERROR_INTERRUPT |
239 GT_CONTEXT_SWITCH_INTERRUPT |
240 GT_WAIT_SEMAPHORE_INTERRUPT;
242 dmask = irqs << 16 | irqs;
245 BUILD_BUG_ON(irqs & 0xffff0000);
247 /* Enable RCS, BCS, VCS and VECS class interrupts. */
248 intel_uncore_write(uncore, GEN11_RENDER_COPY_INTR_ENABLE, dmask);
249 intel_uncore_write(uncore, GEN11_VCS_VECS_INTR_ENABLE, dmask);
251 intel_uncore_write(uncore, GEN12_CCS_RSVD_INTR_ENABLE, smask);
252 if (HAS_HECI_GSC(gt->i915))
253 intel_uncore_write(uncore, GEN11_GUNIT_CSME_INTR_ENABLE,
256 /* Unmask irqs on RCS, BCS, VCS and VECS engines. */
257 intel_uncore_write(uncore, GEN11_RCS0_RSVD_INTR_MASK, ~smask);
258 intel_uncore_write(uncore, GEN11_BCS_RSVD_INTR_MASK, ~smask);
259 if (HAS_ENGINE(gt, BCS1) || HAS_ENGINE(gt, BCS2))
260 intel_uncore_write(uncore, XEHPC_BCS1_BCS2_INTR_MASK, ~dmask);
261 if (HAS_ENGINE(gt, BCS3) || HAS_ENGINE(gt, BCS4))
262 intel_uncore_write(uncore, XEHPC_BCS3_BCS4_INTR_MASK, ~dmask);
263 if (HAS_ENGINE(gt, BCS5) || HAS_ENGINE(gt, BCS6))
264 intel_uncore_write(uncore, XEHPC_BCS5_BCS6_INTR_MASK, ~dmask);
265 if (HAS_ENGINE(gt, BCS7) || HAS_ENGINE(gt, BCS8))
266 intel_uncore_write(uncore, XEHPC_BCS7_BCS8_INTR_MASK, ~dmask);
267 intel_uncore_write(uncore, GEN11_VCS0_VCS1_INTR_MASK, ~dmask);
268 intel_uncore_write(uncore, GEN11_VCS2_VCS3_INTR_MASK, ~dmask);
269 if (HAS_ENGINE(gt, VCS4) || HAS_ENGINE(gt, VCS5))
270 intel_uncore_write(uncore, GEN12_VCS4_VCS5_INTR_MASK, ~dmask);
271 if (HAS_ENGINE(gt, VCS6) || HAS_ENGINE(gt, VCS7))
272 intel_uncore_write(uncore, GEN12_VCS6_VCS7_INTR_MASK, ~dmask);
273 intel_uncore_write(uncore, GEN11_VECS0_VECS1_INTR_MASK, ~dmask);
274 if (HAS_ENGINE(gt, VECS2) || HAS_ENGINE(gt, VECS3))
275 intel_uncore_write(uncore, GEN12_VECS2_VECS3_INTR_MASK, ~dmask);
276 if (HAS_ENGINE(gt, CCS0) || HAS_ENGINE(gt, CCS1))
277 intel_uncore_write(uncore, GEN12_CCS0_CCS1_INTR_MASK, ~dmask);
278 if (HAS_ENGINE(gt, CCS2) || HAS_ENGINE(gt, CCS3))
279 intel_uncore_write(uncore, GEN12_CCS2_CCS3_INTR_MASK, ~dmask);
280 if (HAS_HECI_GSC(gt->i915))
281 intel_uncore_write(uncore, GEN11_GUNIT_CSME_INTR_MASK, ~gsc_mask);
284 * RPS interrupts will get enabled/disabled on demand when RPS itself
285 * is enabled/disabled.
288 gt->pm_imr = ~gt->pm_ier;
289 intel_uncore_write(uncore, GEN11_GPM_WGBOXPERF_INTR_ENABLE, 0);
290 intel_uncore_write(uncore, GEN11_GPM_WGBOXPERF_INTR_MASK, ~0);
292 /* Same thing for GuC interrupts */
293 intel_uncore_write(uncore, GEN11_GUC_SG_INTR_ENABLE, 0);
294 intel_uncore_write(uncore, GEN11_GUC_SG_INTR_MASK, ~0);
297 void gen5_gt_irq_handler(struct intel_gt *gt, u32 gt_iir)
299 if (gt_iir & GT_RENDER_USER_INTERRUPT)
300 intel_engine_cs_irq(gt->engine_class[RENDER_CLASS][0],
303 if (gt_iir & ILK_BSD_USER_INTERRUPT)
304 intel_engine_cs_irq(gt->engine_class[VIDEO_DECODE_CLASS][0],
308 static void gen7_parity_error_irq_handler(struct intel_gt *gt, u32 iir)
310 if (!HAS_L3_DPF(gt->i915))
313 spin_lock(>->irq_lock);
314 gen5_gt_disable_irq(gt, GT_PARITY_ERROR(gt->i915));
315 spin_unlock(>->irq_lock);
317 if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT_S1)
318 gt->i915->l3_parity.which_slice |= 1 << 1;
320 if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT)
321 gt->i915->l3_parity.which_slice |= 1 << 0;
323 schedule_work(>->i915->l3_parity.error_work);
326 void gen6_gt_irq_handler(struct intel_gt *gt, u32 gt_iir)
328 if (gt_iir & GT_RENDER_USER_INTERRUPT)
329 intel_engine_cs_irq(gt->engine_class[RENDER_CLASS][0],
332 if (gt_iir & GT_BSD_USER_INTERRUPT)
333 intel_engine_cs_irq(gt->engine_class[VIDEO_DECODE_CLASS][0],
336 if (gt_iir & GT_BLT_USER_INTERRUPT)
337 intel_engine_cs_irq(gt->engine_class[COPY_ENGINE_CLASS][0],
340 if (gt_iir & (GT_BLT_CS_ERROR_INTERRUPT |
341 GT_BSD_CS_ERROR_INTERRUPT |
342 GT_CS_MASTER_ERROR_INTERRUPT))
343 DRM_DEBUG("Command parser error, gt_iir 0x%08x\n", gt_iir);
345 if (gt_iir & GT_PARITY_ERROR(gt->i915))
346 gen7_parity_error_irq_handler(gt, gt_iir);
349 void gen8_gt_irq_handler(struct intel_gt *gt, u32 master_ctl)
351 void __iomem * const regs = gt->uncore->regs;
354 if (master_ctl & (GEN8_GT_RCS_IRQ | GEN8_GT_BCS_IRQ)) {
355 iir = raw_reg_read(regs, GEN8_GT_IIR(0));
357 intel_engine_cs_irq(gt->engine_class[RENDER_CLASS][0],
358 iir >> GEN8_RCS_IRQ_SHIFT);
359 intel_engine_cs_irq(gt->engine_class[COPY_ENGINE_CLASS][0],
360 iir >> GEN8_BCS_IRQ_SHIFT);
361 raw_reg_write(regs, GEN8_GT_IIR(0), iir);
365 if (master_ctl & (GEN8_GT_VCS0_IRQ | GEN8_GT_VCS1_IRQ)) {
366 iir = raw_reg_read(regs, GEN8_GT_IIR(1));
368 intel_engine_cs_irq(gt->engine_class[VIDEO_DECODE_CLASS][0],
369 iir >> GEN8_VCS0_IRQ_SHIFT);
370 intel_engine_cs_irq(gt->engine_class[VIDEO_DECODE_CLASS][1],
371 iir >> GEN8_VCS1_IRQ_SHIFT);
372 raw_reg_write(regs, GEN8_GT_IIR(1), iir);
376 if (master_ctl & GEN8_GT_VECS_IRQ) {
377 iir = raw_reg_read(regs, GEN8_GT_IIR(3));
379 intel_engine_cs_irq(gt->engine_class[VIDEO_ENHANCEMENT_CLASS][0],
380 iir >> GEN8_VECS_IRQ_SHIFT);
381 raw_reg_write(regs, GEN8_GT_IIR(3), iir);
385 if (master_ctl & (GEN8_GT_PM_IRQ | GEN8_GT_GUC_IRQ)) {
386 iir = raw_reg_read(regs, GEN8_GT_IIR(2));
388 gen6_rps_irq_handler(>->rps, iir);
389 guc_irq_handler(>->uc.guc, iir >> 16);
390 raw_reg_write(regs, GEN8_GT_IIR(2), iir);
395 void gen8_gt_irq_reset(struct intel_gt *gt)
397 struct intel_uncore *uncore = gt->uncore;
399 GEN8_IRQ_RESET_NDX(uncore, GT, 0);
400 GEN8_IRQ_RESET_NDX(uncore, GT, 1);
401 GEN8_IRQ_RESET_NDX(uncore, GT, 2);
402 GEN8_IRQ_RESET_NDX(uncore, GT, 3);
405 void gen8_gt_irq_postinstall(struct intel_gt *gt)
407 /* These are interrupts we'll toggle with the ring mask register */
409 GT_CS_MASTER_ERROR_INTERRUPT |
410 GT_RENDER_USER_INTERRUPT |
411 GT_CONTEXT_SWITCH_INTERRUPT |
412 GT_WAIT_SEMAPHORE_INTERRUPT;
413 const u32 gt_interrupts[] = {
414 irqs << GEN8_RCS_IRQ_SHIFT | irqs << GEN8_BCS_IRQ_SHIFT,
415 irqs << GEN8_VCS0_IRQ_SHIFT | irqs << GEN8_VCS1_IRQ_SHIFT,
417 irqs << GEN8_VECS_IRQ_SHIFT,
419 struct intel_uncore *uncore = gt->uncore;
422 gt->pm_imr = ~gt->pm_ier;
423 GEN8_IRQ_INIT_NDX(uncore, GT, 0, ~gt_interrupts[0], gt_interrupts[0]);
424 GEN8_IRQ_INIT_NDX(uncore, GT, 1, ~gt_interrupts[1], gt_interrupts[1]);
426 * RPS interrupts will get enabled/disabled on demand when RPS itself
427 * is enabled/disabled. Same wil be the case for GuC interrupts.
429 GEN8_IRQ_INIT_NDX(uncore, GT, 2, gt->pm_imr, gt->pm_ier);
430 GEN8_IRQ_INIT_NDX(uncore, GT, 3, ~gt_interrupts[3], gt_interrupts[3]);
433 static void gen5_gt_update_irq(struct intel_gt *gt,
435 u32 enabled_irq_mask)
437 lockdep_assert_held(>->irq_lock);
439 GEM_BUG_ON(enabled_irq_mask & ~interrupt_mask);
441 gt->gt_imr &= ~interrupt_mask;
442 gt->gt_imr |= (~enabled_irq_mask & interrupt_mask);
443 intel_uncore_write(gt->uncore, GTIMR, gt->gt_imr);
446 void gen5_gt_enable_irq(struct intel_gt *gt, u32 mask)
448 gen5_gt_update_irq(gt, mask, mask);
449 intel_uncore_posting_read_fw(gt->uncore, GTIMR);
452 void gen5_gt_disable_irq(struct intel_gt *gt, u32 mask)
454 gen5_gt_update_irq(gt, mask, 0);
457 void gen5_gt_irq_reset(struct intel_gt *gt)
459 struct intel_uncore *uncore = gt->uncore;
461 GEN3_IRQ_RESET(uncore, GT);
462 if (GRAPHICS_VER(gt->i915) >= 6)
463 GEN3_IRQ_RESET(uncore, GEN6_PM);
466 void gen5_gt_irq_postinstall(struct intel_gt *gt)
468 struct intel_uncore *uncore = gt->uncore;
473 if (HAS_L3_DPF(gt->i915)) {
474 /* L3 parity interrupt is always unmasked. */
475 gt->gt_imr = ~GT_PARITY_ERROR(gt->i915);
476 gt_irqs |= GT_PARITY_ERROR(gt->i915);
479 gt_irqs |= GT_RENDER_USER_INTERRUPT;
480 if (GRAPHICS_VER(gt->i915) == 5)
481 gt_irqs |= ILK_BSD_USER_INTERRUPT;
483 gt_irqs |= GT_BLT_USER_INTERRUPT | GT_BSD_USER_INTERRUPT;
485 GEN3_IRQ_INIT(uncore, GT, gt->gt_imr, gt_irqs);
487 if (GRAPHICS_VER(gt->i915) >= 6) {
489 * RPS interrupts will get enabled/disabled on demand when RPS
490 * itself is enabled/disabled.
492 if (HAS_ENGINE(gt, VECS0)) {
493 pm_irqs |= PM_VEBOX_USER_INTERRUPT;
494 gt->pm_ier |= PM_VEBOX_USER_INTERRUPT;
497 gt->pm_imr = 0xffffffff;
498 GEN3_IRQ_INIT(uncore, GEN6_PM, gt->pm_imr, pm_irqs);