drivers/gpu/drm/amd/amdgpu/vega10_ih.c

   1 /*
   2  * Copyright 2016 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23 #include <drm/drmP.h>
  24 #include "amdgpu.h"
  25 #include "amdgpu_ih.h"
  26 #include "soc15.h"
  27
  28 #include "oss/osssys_4_0_offset.h"
  29 #include "oss/osssys_4_0_sh_mask.h"
  30
  31 #include "soc15_common.h"
  32 #include "vega10_ih.h"
  33
  34 #define MAX_REARM_RETRY 10
  35
  36 static void vega10_ih_set_interrupt_funcs(struct amdgpu_device *adev);
  37
  38 /**
  39  * vega10_ih_enable_interrupts - Enable the interrupt ring buffer
  40  *
  41  * @adev: amdgpu_device pointer
  42  *
  43  * Enable the interrupt ring buffer (VEGA10).
  44  */
  45 static void vega10_ih_enable_interrupts(struct amdgpu_device *adev)
  46 {
  47         u32 ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL);
  48
  49         ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_ENABLE, 1);
  50         ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, ENABLE_INTR, 1);
  51         if (amdgpu_virt_support_psp_prg_ih_reg(adev)) {
  52                 if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL, ih_rb_cntl)) {
  53                         DRM_ERROR("PSP program IH_RB_CNTL failed!\n");
  54                         return;
  55                 }
  56         } else {
  57                 WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl);
  58         }
  59         adev->irq.ih.enabled = true;
  60
  61         if (adev->irq.ih1.ring_size) {
  62                 ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1);
  63                 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL_RING1,
  64                                            RB_ENABLE, 1);
  65                 if (amdgpu_virt_support_psp_prg_ih_reg(adev)) {
  66                         if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL_RING1,
  67                                                 ih_rb_cntl)) {
  68                                 DRM_ERROR("program IH_RB_CNTL_RING1 failed!\n");
  69                                 return;
  70                         }
  71                 } else {
  72                         WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1, ih_rb_cntl);
  73                 }
  74                 adev->irq.ih1.enabled = true;
  75         }
  76
  77         if (adev->irq.ih2.ring_size) {
  78                 ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2);
  79                 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL_RING2,
  80                                            RB_ENABLE, 1);
  81                 if (amdgpu_virt_support_psp_prg_ih_reg(adev)) {
  82                         if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL_RING2,
  83                                                 ih_rb_cntl)) {
  84                                 DRM_ERROR("program IH_RB_CNTL_RING2 failed!\n");
  85                                 return;
  86                         }
  87                 } else {
  88                         WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2, ih_rb_cntl);
  89                 }
  90                 adev->irq.ih2.enabled = true;
  91         }
  92 }
  93
  94 /**
  95  * vega10_ih_disable_interrupts - Disable the interrupt ring buffer
  96  *
  97  * @adev: amdgpu_device pointer
  98  *
  99  * Disable the interrupt ring buffer (VEGA10).
 100  */
 101 static void vega10_ih_disable_interrupts(struct amdgpu_device *adev)
 102 {
 103         u32 ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL);
 104
 105         ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_ENABLE, 0);
 106         ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, ENABLE_INTR, 0);
 107         if (amdgpu_virt_support_psp_prg_ih_reg(adev)) {
 108                 if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL, ih_rb_cntl)) {
 109                         DRM_ERROR("PSP program IH_RB_CNTL failed!\n");
 110                         return;
 111                 }
 112         } else {
 113                 WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl);
 114         }
 115
 116         /* set rptr, wptr to 0 */
 117         WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR, 0);
 118         WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR, 0);
 119         adev->irq.ih.enabled = false;
 120         adev->irq.ih.rptr = 0;
 121
 122         if (adev->irq.ih1.ring_size) {
 123                 ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1);
 124                 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL_RING1,
 125                                            RB_ENABLE, 0);
 126                 if (amdgpu_virt_support_psp_prg_ih_reg(adev)) {
 127                         if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL_RING1,
 128                                                 ih_rb_cntl)) {
 129                                 DRM_ERROR("program IH_RB_CNTL_RING1 failed!\n");
 130                                 return;
 131                         }
 132                 } else {
 133                         WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1, ih_rb_cntl);
 134                 }
 135                 /* set rptr, wptr to 0 */
 136                 WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR_RING1, 0);
 137                 WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_RING1, 0);
 138                 adev->irq.ih1.enabled = false;
 139                 adev->irq.ih1.rptr = 0;
 140         }
 141
 142         if (adev->irq.ih2.ring_size) {
 143                 ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2);
 144                 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL_RING2,
 145                                            RB_ENABLE, 0);
 146                 if (amdgpu_virt_support_psp_prg_ih_reg(adev)) {
 147                         if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL_RING2,
 148                                                 ih_rb_cntl)) {
 149                                 DRM_ERROR("program IH_RB_CNTL_RING2 failed!\n");
 150                                 return;
 151                         }
 152                 } else {
 153                         WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2, ih_rb_cntl);
 154                 }
 155
 156                 /* set rptr, wptr to 0 */
 157                 WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR_RING2, 0);
 158                 WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_RING2, 0);
 159                 adev->irq.ih2.enabled = false;
 160                 adev->irq.ih2.rptr = 0;
 161         }
 162 }
 163
 164 static uint32_t vega10_ih_rb_cntl(struct amdgpu_ih_ring *ih, uint32_t ih_rb_cntl)
 165 {
 166         int rb_bufsz = order_base_2(ih->ring_size / 4);
 167
 168         ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL,
 169                                    MC_SPACE, ih->use_bus_addr ? 1 : 4);
 170         ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL,
 171                                    WPTR_OVERFLOW_CLEAR, 1);
 172         ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL,
 173                                    WPTR_OVERFLOW_ENABLE, 1);
 174         ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_SIZE, rb_bufsz);
 175         /* Ring Buffer write pointer writeback. If enabled, IH_RB_WPTR register
 176          * value is written to memory
 177          */
 178         ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL,
 179                                    WPTR_WRITEBACK_ENABLE, 1);
 180         ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_SNOOP, 1);
 181         ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_RO, 0);
 182         ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_VMID, 0);
 183
 184         return ih_rb_cntl;
 185 }
 186
 187 static uint32_t vega10_ih_doorbell_rptr(struct amdgpu_ih_ring *ih)
 188 {
 189         u32 ih_doorbell_rtpr = 0;
 190
 191         if (ih->use_doorbell) {
 192                 ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr,
 193                                                  IH_DOORBELL_RPTR, OFFSET,
 194                                                  ih->doorbell_index);
 195                 ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr,
 196                                                  IH_DOORBELL_RPTR,
 197                                                  ENABLE, 1);
 198         } else {
 199                 ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr,
 200                                                  IH_DOORBELL_RPTR,
 201                                                  ENABLE, 0);
 202         }
 203         return ih_doorbell_rtpr;
 204 }
 205
 206 /**
 207  * vega10_ih_irq_init - init and enable the interrupt ring
 208  *
 209  * @adev: amdgpu_device pointer
 210  *
 211  * Allocate a ring buffer for the interrupt controller,
 212  * enable the RLC, disable interrupts, enable the IH
 213  * ring buffer and enable it (VI).
 214  * Called at device load and reume.
 215  * Returns 0 for success, errors for failure.
 216  */
 217 static int vega10_ih_irq_init(struct amdgpu_device *adev)
 218 {
 219         struct amdgpu_ih_ring *ih;
 220         u32 ih_rb_cntl;
 221         int ret = 0;
 222         u32 tmp;
 223
 224         /* disable irqs */
 225         vega10_ih_disable_interrupts(adev);
 226
 227         adev->nbio_funcs->ih_control(adev);
 228
 229         ih = &adev->irq.ih;
 230         /* Ring Buffer base. [39:8] of 40-bit address of the beginning of the ring buffer*/
 231         WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE, ih->gpu_addr >> 8);
 232         WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_HI, (ih->gpu_addr >> 40) & 0xff);
 233
 234         ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL);
 235         ih_rb_cntl = vega10_ih_rb_cntl(ih, ih_rb_cntl);
 236         ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RPTR_REARM,
 237                                    !!adev->irq.msi_enabled);
 238
 239         if (amdgpu_virt_support_psp_prg_ih_reg(adev)) {
 240                 if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL, ih_rb_cntl)) {
 241                         DRM_ERROR("PSP program IH_RB_CNTL failed!\n");
 242                         return -ETIMEDOUT;
 243                 }
 244         } else {
 245                 WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl);
 246         }
 247
 248         /* set the writeback address whether it's enabled or not */
 249         WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_ADDR_LO,
 250                      lower_32_bits(ih->wptr_addr));
 251         WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_ADDR_HI,
 252                      upper_32_bits(ih->wptr_addr) & 0xFFFF);
 253
 254         /* set rptr, wptr to 0 */
 255         WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR, 0);
 256         WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR, 0);
 257
 258         WREG32_SOC15(OSSSYS, 0, mmIH_DOORBELL_RPTR,
 259                      vega10_ih_doorbell_rptr(ih));
 260
 261         ih = &adev->irq.ih1;
 262         if (ih->ring_size) {
 263                 WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_RING1, ih->gpu_addr >> 8);
 264                 WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_HI_RING1,
 265                              (ih->gpu_addr >> 40) & 0xff);
 266
 267                 ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1);
 268                 ih_rb_cntl = vega10_ih_rb_cntl(ih, ih_rb_cntl);
 269                 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL,
 270                                            WPTR_OVERFLOW_ENABLE, 0);
 271                 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL,
 272                                            RB_FULL_DRAIN_ENABLE, 1);
 273                 if (amdgpu_virt_support_psp_prg_ih_reg(adev)) {
 274                         if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL_RING1,
 275                                                 ih_rb_cntl)) {
 276                                 DRM_ERROR("program IH_RB_CNTL_RING1 failed!\n");
 277                                 return -ETIMEDOUT;
 278                         }
 279                 } else {
 280                         WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1, ih_rb_cntl);
 281                 }
 282
 283                 /* set rptr, wptr to 0 */
 284                 WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_RING1, 0);
 285                 WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR_RING1, 0);
 286
 287                 WREG32_SOC15(OSSSYS, 0, mmIH_DOORBELL_RPTR_RING1,
 288                              vega10_ih_doorbell_rptr(ih));
 289         }
 290
 291         ih = &adev->irq.ih2;
 292         if (ih->ring_size) {
 293                 WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_RING2, ih->gpu_addr >> 8);
 294                 WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_HI_RING2,
 295                              (ih->gpu_addr >> 40) & 0xff);
 296
 297                 ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2);
 298                 ih_rb_cntl = vega10_ih_rb_cntl(ih, ih_rb_cntl);
 299
 300                 if (amdgpu_virt_support_psp_prg_ih_reg(adev)) {
 301                         if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL_RING2,
 302                                                 ih_rb_cntl)) {
 303                                 DRM_ERROR("program IH_RB_CNTL_RING2 failed!\n");
 304                                 return -ETIMEDOUT;
 305                         }
 306                 } else {
 307                         WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2, ih_rb_cntl);
 308                 }
 309
 310                 /* set rptr, wptr to 0 */
 311                 WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_RING2, 0);
 312                 WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR_RING2, 0);
 313
 314                 WREG32_SOC15(OSSSYS, 0, mmIH_DOORBELL_RPTR_RING2,
 315                              vega10_ih_doorbell_rptr(ih));
 316         }
 317
 318         tmp = RREG32_SOC15(OSSSYS, 0, mmIH_STORM_CLIENT_LIST_CNTL);
 319         tmp = REG_SET_FIELD(tmp, IH_STORM_CLIENT_LIST_CNTL,
 320                             CLIENT18_IS_STORM_CLIENT, 1);
 321         WREG32_SOC15(OSSSYS, 0, mmIH_STORM_CLIENT_LIST_CNTL, tmp);
 322
 323         tmp = RREG32_SOC15(OSSSYS, 0, mmIH_INT_FLOOD_CNTL);
 324         tmp = REG_SET_FIELD(tmp, IH_INT_FLOOD_CNTL, FLOOD_CNTL_ENABLE, 1);
 325         WREG32_SOC15(OSSSYS, 0, mmIH_INT_FLOOD_CNTL, tmp);
 326
 327         pci_set_master(adev->pdev);
 328
 329         /* enable interrupts */
 330         vega10_ih_enable_interrupts(adev);
 331
 332         return ret;
 333 }
 334
 335 /**
 336  * vega10_ih_irq_disable - disable interrupts
 337  *
 338  * @adev: amdgpu_device pointer
 339  *
 340  * Disable interrupts on the hw (VEGA10).
 341  */
 342 static void vega10_ih_irq_disable(struct amdgpu_device *adev)
 343 {
 344         vega10_ih_disable_interrupts(adev);
 345
 346         /* Wait and acknowledge irq */
 347         mdelay(1);
 348 }
 349
 350 /**
 351  * vega10_ih_get_wptr - get the IH ring buffer wptr
 352  *
 353  * @adev: amdgpu_device pointer
 354  *
 355  * Get the IH ring buffer wptr from either the register
 356  * or the writeback memory buffer (VEGA10).  Also check for
 357  * ring buffer overflow and deal with it.
 358  * Returns the value of the wptr.
 359  */
 360 static u32 vega10_ih_get_wptr(struct amdgpu_device *adev,
 361                               struct amdgpu_ih_ring *ih)
 362 {
 363         u32 wptr, reg, tmp;
 364
 365         wptr = le32_to_cpu(*ih->wptr_cpu);
 366
 367         if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW))
 368                 goto out;
 369
 370         /* Double check that the overflow wasn't already cleared. */
 371
 372         if (ih == &adev->irq.ih)
 373                 reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR);
 374         else if (ih == &adev->irq.ih1)
 375                 reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_RING1);
 376         else if (ih == &adev->irq.ih2)
 377                 reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_RING2);
 378         else
 379                 BUG();
 380
 381         wptr = RREG32_NO_KIQ(reg);
 382         if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW))
 383                 goto out;
 384
 385         wptr = REG_SET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW, 0);
 386
 387         /* When a ring buffer overflow happen start parsing interrupt
 388          * from the last not overwritten vector (wptr + 32). Hopefully
 389          * this should allow us to catchup.
 390          */
 391         tmp = (wptr + 32) & ih->ptr_mask;
 392         dev_warn(adev->dev, "IH ring buffer overflow "
 393                  "(0x%08X, 0x%08X, 0x%08X)\n",
 394                  wptr, ih->rptr, tmp);
 395         ih->rptr = tmp;
 396
 397         if (ih == &adev->irq.ih)
 398                 reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL);
 399         else if (ih == &adev->irq.ih1)
 400                 reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL_RING1);
 401         else if (ih == &adev->irq.ih2)
 402                 reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL_RING2);
 403         else
 404                 BUG();
 405
 406         tmp = RREG32_NO_KIQ(reg);
 407         tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1);
 408         WREG32_NO_KIQ(reg, tmp);
 409
 410 out:
 411         return (wptr & ih->ptr_mask);
 412 }
 413
 414 /**
 415  * vega10_ih_decode_iv - decode an interrupt vector
 416  *
 417  * @adev: amdgpu_device pointer
 418  *
 419  * Decodes the interrupt vector at the current rptr
 420  * position and also advance the position.
 421  */
 422 static void vega10_ih_decode_iv(struct amdgpu_device *adev,
 423                                 struct amdgpu_ih_ring *ih,
 424                                 struct amdgpu_iv_entry *entry)
 425 {
 426         /* wptr/rptr are in bytes! */
 427         u32 ring_index = ih->rptr >> 2;
 428         uint32_t dw[8];
 429
 430         dw[0] = le32_to_cpu(ih->ring[ring_index + 0]);
 431         dw[1] = le32_to_cpu(ih->ring[ring_index + 1]);
 432         dw[2] = le32_to_cpu(ih->ring[ring_index + 2]);
 433         dw[3] = le32_to_cpu(ih->ring[ring_index + 3]);
 434         dw[4] = le32_to_cpu(ih->ring[ring_index + 4]);
 435         dw[5] = le32_to_cpu(ih->ring[ring_index + 5]);
 436         dw[6] = le32_to_cpu(ih->ring[ring_index + 6]);
 437         dw[7] = le32_to_cpu(ih->ring[ring_index + 7]);
 438
 439         entry->client_id = dw[0] & 0xff;
 440         entry->src_id = (dw[0] >> 8) & 0xff;
 441         entry->ring_id = (dw[0] >> 16) & 0xff;
 442         entry->vmid = (dw[0] >> 24) & 0xf;
 443         entry->vmid_src = (dw[0] >> 31);
 444         entry->timestamp = dw[1] | ((u64)(dw[2] & 0xffff) << 32);
 445         entry->timestamp_src = dw[2] >> 31;
 446         entry->pasid = dw[3] & 0xffff;
 447         entry->pasid_src = dw[3] >> 31;
 448         entry->src_data[0] = dw[4];
 449         entry->src_data[1] = dw[5];
 450         entry->src_data[2] = dw[6];
 451         entry->src_data[3] = dw[7];
 452
 453         /* wptr/rptr are in bytes! */
 454         ih->rptr += 32;
 455 }
 456
 457 /**
 458  * vega10_ih_irq_rearm - rearm IRQ if lost
 459  *
 460  * @adev: amdgpu_device pointer
 461  *
 462  */
 463 static void vega10_ih_irq_rearm(struct amdgpu_device *adev,
 464                                struct amdgpu_ih_ring *ih)
 465 {
 466         uint32_t reg_rptr = 0;
 467         uint32_t v = 0;
 468         uint32_t i = 0;
 469
 470         if (ih == &adev->irq.ih)
 471                 reg_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR);
 472         else if (ih == &adev->irq.ih1)
 473                 reg_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR_RING1);
 474         else if (ih == &adev->irq.ih2)
 475                 reg_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR_RING2);
 476         else
 477                 return;
 478
 479         /* Rearm IRQ / re-wwrite doorbell if doorbell write is lost */
 480         for (i = 0; i < MAX_REARM_RETRY; i++) {
 481                 v = RREG32_NO_KIQ(reg_rptr);
 482                 if ((v < ih->ring_size) && (v != ih->rptr))
 483                         WDOORBELL32(ih->doorbell_index, ih->rptr);
 484                 else
 485                         break;
 486         }
 487 }
 488
 489 /**
 490  * vega10_ih_set_rptr - set the IH ring buffer rptr
 491  *
 492  * @adev: amdgpu_device pointer
 493  *
 494  * Set the IH ring buffer rptr.
 495  */
 496 static void vega10_ih_set_rptr(struct amdgpu_device *adev,
 497                                struct amdgpu_ih_ring *ih)
 498 {
 499         if (ih->use_doorbell) {
 500                 /* XXX check if swapping is necessary on BE */
 501                 *ih->rptr_cpu = ih->rptr;
 502                 WDOORBELL32(ih->doorbell_index, ih->rptr);
 503
 504                 if (amdgpu_sriov_vf(adev))
 505                         vega10_ih_irq_rearm(adev, ih);
 506         } else if (ih == &adev->irq.ih) {
 507                 WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR, ih->rptr);
 508         } else if (ih == &adev->irq.ih1) {
 509                 WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR_RING1, ih->rptr);
 510         } else if (ih == &adev->irq.ih2) {
 511                 WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR_RING2, ih->rptr);
 512         }
 513 }
 514
 515 /**
 516  * vega10_ih_self_irq - dispatch work for ring 1 and 2
 517  *
 518  * @adev: amdgpu_device pointer
 519  * @source: irq source
 520  * @entry: IV with WPTR update
 521  *
 522  * Update the WPTR from the IV and schedule work to handle the entries.
 523  */
 524 static int vega10_ih_self_irq(struct amdgpu_device *adev,
 525                               struct amdgpu_irq_src *source,
 526                               struct amdgpu_iv_entry *entry)
 527 {
 528         uint32_t wptr = cpu_to_le32(entry->src_data[0]);
 529
 530         switch (entry->ring_id) {
 531         case 1:
 532                 *adev->irq.ih1.wptr_cpu = wptr;
 533                 schedule_work(&adev->irq.ih1_work);
 534                 break;
 535         case 2:
 536                 *adev->irq.ih2.wptr_cpu = wptr;
 537                 schedule_work(&adev->irq.ih2_work);
 538                 break;
 539         default: break;
 540         }
 541         return 0;
 542 }
 543
 544 static const struct amdgpu_irq_src_funcs vega10_ih_self_irq_funcs = {
 545         .process = vega10_ih_self_irq,
 546 };
 547
 548 static void vega10_ih_set_self_irq_funcs(struct amdgpu_device *adev)
 549 {
 550         adev->irq.self_irq.num_types = 0;
 551         adev->irq.self_irq.funcs = &vega10_ih_self_irq_funcs;
 552 }
 553
 554 static int vega10_ih_early_init(void *handle)
 555 {
 556         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 557
 558         vega10_ih_set_interrupt_funcs(adev);
 559         vega10_ih_set_self_irq_funcs(adev);
 560         return 0;
 561 }
 562
 563 static int vega10_ih_sw_init(void *handle)
 564 {
 565         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 566         int r;
 567
 568         r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_IH, 0,
 569                               &adev->irq.self_irq);
 570         if (r)
 571                 return r;
 572
 573         r = amdgpu_ih_ring_init(adev, &adev->irq.ih, 256 * 1024, true);
 574         if (r)
 575                 return r;
 576
 577         adev->irq.ih.use_doorbell = true;
 578         adev->irq.ih.doorbell_index = adev->doorbell_index.ih << 1;
 579
 580         r = amdgpu_ih_ring_init(adev, &adev->irq.ih1, PAGE_SIZE, true);
 581         if (r)
 582                 return r;
 583
 584         adev->irq.ih1.use_doorbell = true;
 585         adev->irq.ih1.doorbell_index = (adev->doorbell_index.ih + 1) << 1;
 586
 587         r = amdgpu_ih_ring_init(adev, &adev->irq.ih2, PAGE_SIZE, true);
 588         if (r)
 589                 return r;
 590
 591         adev->irq.ih2.use_doorbell = true;
 592         adev->irq.ih2.doorbell_index = (adev->doorbell_index.ih + 2) << 1;
 593
 594         r = amdgpu_irq_init(adev);
 595
 596         return r;
 597 }
 598
 599 static int vega10_ih_sw_fini(void *handle)
 600 {
 601         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 602
 603         amdgpu_irq_fini(adev);
 604         amdgpu_ih_ring_fini(adev, &adev->irq.ih2);
 605         amdgpu_ih_ring_fini(adev, &adev->irq.ih1);
 606         amdgpu_ih_ring_fini(adev, &adev->irq.ih);
 607
 608         return 0;
 609 }
 610
 611 static int vega10_ih_hw_init(void *handle)
 612 {
 613         int r;
 614         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 615
 616         r = vega10_ih_irq_init(adev);
 617         if (r)
 618                 return r;
 619
 620         return 0;
 621 }
 622
 623 static int vega10_ih_hw_fini(void *handle)
 624 {
 625         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 626
 627         vega10_ih_irq_disable(adev);
 628
 629         return 0;
 630 }
 631
 632 static int vega10_ih_suspend(void *handle)
 633 {
 634         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 635
 636         return vega10_ih_hw_fini(adev);
 637 }
 638
 639 static int vega10_ih_resume(void *handle)
 640 {
 641         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 642
 643         return vega10_ih_hw_init(adev);
 644 }
 645
 646 static bool vega10_ih_is_idle(void *handle)
 647 {
 648         /* todo */
 649         return true;
 650 }
 651
 652 static int vega10_ih_wait_for_idle(void *handle)
 653 {
 654         /* todo */
 655         return -ETIMEDOUT;
 656 }
 657
 658 static int vega10_ih_soft_reset(void *handle)
 659 {
 660         /* todo */
 661
 662         return 0;
 663 }
 664
 665 static int vega10_ih_set_clockgating_state(void *handle,
 666                                           enum amd_clockgating_state state)
 667 {
 668         return 0;
 669 }
 670
 671 static int vega10_ih_set_powergating_state(void *handle,
 672                                           enum amd_powergating_state state)
 673 {
 674         return 0;
 675 }
 676
 677 const struct amd_ip_funcs vega10_ih_ip_funcs = {
 678         .name = "vega10_ih",
 679         .early_init = vega10_ih_early_init,
 680         .late_init = NULL,
 681         .sw_init = vega10_ih_sw_init,
 682         .sw_fini = vega10_ih_sw_fini,
 683         .hw_init = vega10_ih_hw_init,
 684         .hw_fini = vega10_ih_hw_fini,
 685         .suspend = vega10_ih_suspend,
 686         .resume = vega10_ih_resume,
 687         .is_idle = vega10_ih_is_idle,
 688         .wait_for_idle = vega10_ih_wait_for_idle,
 689         .soft_reset = vega10_ih_soft_reset,
 690         .set_clockgating_state = vega10_ih_set_clockgating_state,
 691         .set_powergating_state = vega10_ih_set_powergating_state,
 692 };
 693
 694 static const struct amdgpu_ih_funcs vega10_ih_funcs = {
 695         .get_wptr = vega10_ih_get_wptr,
 696         .decode_iv = vega10_ih_decode_iv,
 697         .set_rptr = vega10_ih_set_rptr
 698 };
 699
 700 static void vega10_ih_set_interrupt_funcs(struct amdgpu_device *adev)
 701 {
 702         adev->irq.ih_funcs = &vega10_ih_funcs;
 703 }
 704
 705 const struct amdgpu_ip_block_version vega10_ih_ip_block =
 706 {
 707         .type = AMD_IP_BLOCK_TYPE_IH,
 708         .major = 4,
 709         .minor = 0,
 710         .rev = 0,
 711         .funcs = &vega10_ih_ip_funcs,
 712 };