2 * Copyright 2016 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
25 #include "amdgpu_ih.h"
28 #include "oss/osssys_4_0_offset.h"
29 #include "oss/osssys_4_0_sh_mask.h"
31 #include "soc15_common.h"
32 #include "vega10_ih.h"
36 static void vega10_ih_set_interrupt_funcs(struct amdgpu_device *adev);
39 * vega10_ih_enable_interrupts - Enable the interrupt ring buffer
41 * @adev: amdgpu_device pointer
43 * Enable the interrupt ring buffer (VEGA10).
45 static void vega10_ih_enable_interrupts(struct amdgpu_device *adev)
47 u32 ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL);
49 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_ENABLE, 1);
50 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, ENABLE_INTR, 1);
51 WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl);
52 adev->irq.ih.enabled = true;
54 if (adev->irq.ih1.ring_size) {
55 ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1);
56 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL_RING1,
58 WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1, ih_rb_cntl);
59 adev->irq.ih1.enabled = true;
62 if (adev->irq.ih2.ring_size) {
63 ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2);
64 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL_RING2,
66 WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2, ih_rb_cntl);
67 adev->irq.ih2.enabled = true;
72 * vega10_ih_disable_interrupts - Disable the interrupt ring buffer
74 * @adev: amdgpu_device pointer
76 * Disable the interrupt ring buffer (VEGA10).
78 static void vega10_ih_disable_interrupts(struct amdgpu_device *adev)
80 u32 ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL);
82 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_ENABLE, 0);
83 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, ENABLE_INTR, 0);
84 WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl);
85 /* set rptr, wptr to 0 */
86 WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR, 0);
87 WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR, 0);
88 adev->irq.ih.enabled = false;
89 adev->irq.ih.rptr = 0;
91 if (adev->irq.ih1.ring_size) {
92 ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1);
93 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL_RING1,
95 WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1, ih_rb_cntl);
96 /* set rptr, wptr to 0 */
97 WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR_RING1, 0);
98 WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_RING1, 0);
99 adev->irq.ih1.enabled = false;
100 adev->irq.ih1.rptr = 0;
103 if (adev->irq.ih2.ring_size) {
104 ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2);
105 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL_RING2,
107 WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2, ih_rb_cntl);
108 /* set rptr, wptr to 0 */
109 WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR_RING2, 0);
110 WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_RING2, 0);
111 adev->irq.ih2.enabled = false;
112 adev->irq.ih2.rptr = 0;
116 static uint32_t vega10_ih_rb_cntl(struct amdgpu_ih_ring *ih, uint32_t ih_rb_cntl)
118 int rb_bufsz = order_base_2(ih->ring_size / 4);
120 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL,
121 MC_SPACE, ih->use_bus_addr ? 1 : 4);
122 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL,
123 WPTR_OVERFLOW_CLEAR, 1);
124 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL,
125 WPTR_OVERFLOW_ENABLE, 1);
126 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_SIZE, rb_bufsz);
127 /* Ring Buffer write pointer writeback. If enabled, IH_RB_WPTR register
128 * value is written to memory
130 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL,
131 WPTR_WRITEBACK_ENABLE, 1);
132 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_SNOOP, 1);
133 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_RO, 0);
134 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_VMID, 0);
140 * vega10_ih_irq_init - init and enable the interrupt ring
142 * @adev: amdgpu_device pointer
144 * Allocate a ring buffer for the interrupt controller,
145 * enable the RLC, disable interrupts, enable the IH
146 * ring buffer and enable it (VI).
147 * Called at device load and reume.
148 * Returns 0 for success, errors for failure.
150 static int vega10_ih_irq_init(struct amdgpu_device *adev)
152 struct amdgpu_ih_ring *ih;
154 u32 ih_rb_cntl, ih_doorbell_rtpr;
158 vega10_ih_disable_interrupts(adev);
160 adev->nbio_funcs->ih_control(adev);
163 /* Ring Buffer base. [39:8] of 40-bit address of the beginning of the ring buffer*/
164 WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE, ih->gpu_addr >> 8);
165 WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_HI, (ih->gpu_addr >> 40) & 0xff);
167 ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL);
168 ih_rb_cntl = vega10_ih_rb_cntl(ih, ih_rb_cntl);
169 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RPTR_REARM,
170 !!adev->irq.msi_enabled);
171 WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl);
173 /* set the writeback address whether it's enabled or not */
174 WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_ADDR_LO,
175 lower_32_bits(ih->wptr_addr));
176 WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_ADDR_HI,
177 upper_32_bits(ih->wptr_addr) & 0xFFFF);
179 /* set rptr, wptr to 0 */
180 WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR, 0);
181 WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR, 0);
183 ih_doorbell_rtpr = RREG32_SOC15(OSSSYS, 0, mmIH_DOORBELL_RPTR);
184 if (adev->irq.ih.use_doorbell) {
185 ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr,
186 IH_DOORBELL_RPTR, OFFSET,
187 adev->irq.ih.doorbell_index);
188 ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr,
192 ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr,
196 WREG32_SOC15(OSSSYS, 0, mmIH_DOORBELL_RPTR, ih_doorbell_rtpr);
200 WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_RING1, ih->gpu_addr >> 8);
201 WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_HI_RING1,
202 (ih->gpu_addr >> 40) & 0xff);
204 ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1);
205 ih_rb_cntl = vega10_ih_rb_cntl(ih, ih_rb_cntl);
206 WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1, ih_rb_cntl);
208 /* set rptr, wptr to 0 */
209 WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR_RING1, 0);
210 WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_RING1, 0);
215 WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_RING2, ih->gpu_addr >> 8);
216 WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_HI_RING2,
217 (ih->gpu_addr >> 40) & 0xff);
219 ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1);
220 ih_rb_cntl = vega10_ih_rb_cntl(ih, ih_rb_cntl);
221 WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2, ih_rb_cntl);
223 /* set rptr, wptr to 0 */
224 WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR_RING2, 0);
225 WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_RING2, 0);
228 tmp = RREG32_SOC15(OSSSYS, 0, mmIH_STORM_CLIENT_LIST_CNTL);
229 tmp = REG_SET_FIELD(tmp, IH_STORM_CLIENT_LIST_CNTL,
230 CLIENT18_IS_STORM_CLIENT, 1);
231 WREG32_SOC15(OSSSYS, 0, mmIH_STORM_CLIENT_LIST_CNTL, tmp);
233 tmp = RREG32_SOC15(OSSSYS, 0, mmIH_INT_FLOOD_CNTL);
234 tmp = REG_SET_FIELD(tmp, IH_INT_FLOOD_CNTL, FLOOD_CNTL_ENABLE, 1);
235 WREG32_SOC15(OSSSYS, 0, mmIH_INT_FLOOD_CNTL, tmp);
237 pci_set_master(adev->pdev);
239 /* enable interrupts */
240 vega10_ih_enable_interrupts(adev);
246 * vega10_ih_irq_disable - disable interrupts
248 * @adev: amdgpu_device pointer
250 * Disable interrupts on the hw (VEGA10).
252 static void vega10_ih_irq_disable(struct amdgpu_device *adev)
254 vega10_ih_disable_interrupts(adev);
256 /* Wait and acknowledge irq */
261 * vega10_ih_get_wptr - get the IH ring buffer wptr
263 * @adev: amdgpu_device pointer
265 * Get the IH ring buffer wptr from either the register
266 * or the writeback memory buffer (VEGA10). Also check for
267 * ring buffer overflow and deal with it.
268 * Returns the value of the wptr.
270 static u32 vega10_ih_get_wptr(struct amdgpu_device *adev,
271 struct amdgpu_ih_ring *ih)
275 wptr = le32_to_cpu(*ih->wptr_cpu);
277 if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW))
280 /* Double check that the overflow wasn't already cleared. */
281 wptr = RREG32_NO_KIQ(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR));
282 if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW))
285 wptr = REG_SET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW, 0);
287 /* When a ring buffer overflow happen start parsing interrupt
288 * from the last not overwritten vector (wptr + 32). Hopefully
289 * this should allow us to catchup.
291 tmp = (wptr + 32) & ih->ptr_mask;
292 dev_warn(adev->dev, "IH ring buffer overflow "
293 "(0x%08X, 0x%08X, 0x%08X)\n",
294 wptr, ih->rptr, tmp);
297 tmp = RREG32_NO_KIQ(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL));
298 tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1);
299 WREG32_NO_KIQ(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL), tmp);
302 return (wptr & ih->ptr_mask);
306 * vega10_ih_decode_iv - decode an interrupt vector
308 * @adev: amdgpu_device pointer
310 * Decodes the interrupt vector at the current rptr
311 * position and also advance the position.
313 static void vega10_ih_decode_iv(struct amdgpu_device *adev,
314 struct amdgpu_ih_ring *ih,
315 struct amdgpu_iv_entry *entry)
317 /* wptr/rptr are in bytes! */
318 u32 ring_index = ih->rptr >> 2;
321 dw[0] = le32_to_cpu(ih->ring[ring_index + 0]);
322 dw[1] = le32_to_cpu(ih->ring[ring_index + 1]);
323 dw[2] = le32_to_cpu(ih->ring[ring_index + 2]);
324 dw[3] = le32_to_cpu(ih->ring[ring_index + 3]);
325 dw[4] = le32_to_cpu(ih->ring[ring_index + 4]);
326 dw[5] = le32_to_cpu(ih->ring[ring_index + 5]);
327 dw[6] = le32_to_cpu(ih->ring[ring_index + 6]);
328 dw[7] = le32_to_cpu(ih->ring[ring_index + 7]);
330 entry->client_id = dw[0] & 0xff;
331 entry->src_id = (dw[0] >> 8) & 0xff;
332 entry->ring_id = (dw[0] >> 16) & 0xff;
333 entry->vmid = (dw[0] >> 24) & 0xf;
334 entry->vmid_src = (dw[0] >> 31);
335 entry->timestamp = dw[1] | ((u64)(dw[2] & 0xffff) << 32);
336 entry->timestamp_src = dw[2] >> 31;
337 entry->pasid = dw[3] & 0xffff;
338 entry->pasid_src = dw[3] >> 31;
339 entry->src_data[0] = dw[4];
340 entry->src_data[1] = dw[5];
341 entry->src_data[2] = dw[6];
342 entry->src_data[3] = dw[7];
344 /* wptr/rptr are in bytes! */
349 * vega10_ih_set_rptr - set the IH ring buffer rptr
351 * @adev: amdgpu_device pointer
353 * Set the IH ring buffer rptr.
355 static void vega10_ih_set_rptr(struct amdgpu_device *adev,
356 struct amdgpu_ih_ring *ih)
358 if (ih->use_doorbell) {
359 /* XXX check if swapping is necessary on BE */
360 *ih->rptr_cpu = ih->rptr;
361 WDOORBELL32(ih->doorbell_index, ih->rptr);
363 WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR, ih->rptr);
367 static int vega10_ih_early_init(void *handle)
369 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
371 vega10_ih_set_interrupt_funcs(adev);
375 static int vega10_ih_sw_init(void *handle)
378 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
380 r = amdgpu_ih_ring_init(adev, &adev->irq.ih, 256 * 1024, true);
384 if (adev->asic_type == CHIP_VEGA10) {
385 r = amdgpu_ih_ring_init(adev, &adev->irq.ih1, PAGE_SIZE, true);
389 r = amdgpu_ih_ring_init(adev, &adev->irq.ih2, PAGE_SIZE, true);
394 /* TODO add doorbell for IH1 & IH2 as well */
395 adev->irq.ih.use_doorbell = true;
396 adev->irq.ih.doorbell_index = adev->doorbell_index.ih << 1;
398 r = amdgpu_irq_init(adev);
403 static int vega10_ih_sw_fini(void *handle)
405 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
407 amdgpu_irq_fini(adev);
408 amdgpu_ih_ring_fini(adev, &adev->irq.ih2);
409 amdgpu_ih_ring_fini(adev, &adev->irq.ih1);
410 amdgpu_ih_ring_fini(adev, &adev->irq.ih);
415 static int vega10_ih_hw_init(void *handle)
418 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
420 r = vega10_ih_irq_init(adev);
427 static int vega10_ih_hw_fini(void *handle)
429 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
431 vega10_ih_irq_disable(adev);
436 static int vega10_ih_suspend(void *handle)
438 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
440 return vega10_ih_hw_fini(adev);
443 static int vega10_ih_resume(void *handle)
445 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
447 return vega10_ih_hw_init(adev);
450 static bool vega10_ih_is_idle(void *handle)
456 static int vega10_ih_wait_for_idle(void *handle)
462 static int vega10_ih_soft_reset(void *handle)
469 static int vega10_ih_set_clockgating_state(void *handle,
470 enum amd_clockgating_state state)
475 static int vega10_ih_set_powergating_state(void *handle,
476 enum amd_powergating_state state)
481 const struct amd_ip_funcs vega10_ih_ip_funcs = {
483 .early_init = vega10_ih_early_init,
485 .sw_init = vega10_ih_sw_init,
486 .sw_fini = vega10_ih_sw_fini,
487 .hw_init = vega10_ih_hw_init,
488 .hw_fini = vega10_ih_hw_fini,
489 .suspend = vega10_ih_suspend,
490 .resume = vega10_ih_resume,
491 .is_idle = vega10_ih_is_idle,
492 .wait_for_idle = vega10_ih_wait_for_idle,
493 .soft_reset = vega10_ih_soft_reset,
494 .set_clockgating_state = vega10_ih_set_clockgating_state,
495 .set_powergating_state = vega10_ih_set_powergating_state,
498 static const struct amdgpu_ih_funcs vega10_ih_funcs = {
499 .get_wptr = vega10_ih_get_wptr,
500 .decode_iv = vega10_ih_decode_iv,
501 .set_rptr = vega10_ih_set_rptr
504 static void vega10_ih_set_interrupt_funcs(struct amdgpu_device *adev)
506 adev->irq.ih_funcs = &vega10_ih_funcs;
509 const struct amdgpu_ip_block_version vega10_ih_ip_block =
511 .type = AMD_IP_BLOCK_TYPE_IH,
515 .funcs = &vega10_ih_ip_funcs,