drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c

   1 /*
   2  * Copyright 2019 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23
  24 #include <linux/delay.h>
  25 #include <linux/firmware.h>
  26 #include <linux/module.h>
  27 #include <linux/pci.h>
  28
  29 #include "amdgpu.h"
  30 #include "amdgpu_ucode.h"
  31 #include "amdgpu_trace.h"
  32
  33 #include "gc/gc_10_3_0_offset.h"
  34 #include "gc/gc_10_3_0_sh_mask.h"
  35 #include "ivsrcid/sdma0/irqsrcs_sdma0_5_0.h"
  36 #include "ivsrcid/sdma1/irqsrcs_sdma1_5_0.h"
  37 #include "ivsrcid/sdma2/irqsrcs_sdma2_5_0.h"
  38 #include "ivsrcid/sdma3/irqsrcs_sdma3_5_0.h"
  39
  40 #include "soc15_common.h"
  41 #include "soc15.h"
  42 #include "navi10_sdma_pkt_open.h"
  43 #include "nbio_v2_3.h"
  44 #include "sdma_common.h"
  45 #include "sdma_v5_2.h"
  46
  47 MODULE_FIRMWARE("amdgpu/sienna_cichlid_sdma.bin");
  48 MODULE_FIRMWARE("amdgpu/navy_flounder_sdma.bin");
  49 MODULE_FIRMWARE("amdgpu/dimgrey_cavefish_sdma.bin");
  50 MODULE_FIRMWARE("amdgpu/beige_goby_sdma.bin");
  51
  52 MODULE_FIRMWARE("amdgpu/vangogh_sdma.bin");
  53 MODULE_FIRMWARE("amdgpu/yellow_carp_sdma.bin");
  54 MODULE_FIRMWARE("amdgpu/sdma_5_2_6.bin");
  55 MODULE_FIRMWARE("amdgpu/sdma_5_2_7.bin");
  56
  57 #define SDMA1_REG_OFFSET 0x600
  58 #define SDMA3_REG_OFFSET 0x400
  59 #define SDMA0_HYP_DEC_REG_START 0x5880
  60 #define SDMA0_HYP_DEC_REG_END 0x5893
  61 #define SDMA1_HYP_DEC_REG_OFFSET 0x20
  62
  63 static void sdma_v5_2_set_ring_funcs(struct amdgpu_device *adev);
  64 static void sdma_v5_2_set_buffer_funcs(struct amdgpu_device *adev);
  65 static void sdma_v5_2_set_vm_pte_funcs(struct amdgpu_device *adev);
  66 static void sdma_v5_2_set_irq_funcs(struct amdgpu_device *adev);
  67
  68 static u32 sdma_v5_2_get_reg_offset(struct amdgpu_device *adev, u32 instance, u32 internal_offset)
  69 {
  70         u32 base;
  71
  72         if (internal_offset >= SDMA0_HYP_DEC_REG_START &&
  73             internal_offset <= SDMA0_HYP_DEC_REG_END) {
  74                 base = adev->reg_offset[GC_HWIP][0][1];
  75                 if (instance != 0)
  76                         internal_offset += SDMA1_HYP_DEC_REG_OFFSET * instance;
  77         } else {
  78                 if (instance < 2) {
  79                         base = adev->reg_offset[GC_HWIP][0][0];
  80                         if (instance == 1)
  81                                 internal_offset += SDMA1_REG_OFFSET;
  82                 } else {
  83                         base = adev->reg_offset[GC_HWIP][0][2];
  84                         if (instance == 3)
  85                                 internal_offset += SDMA3_REG_OFFSET;
  86                 }
  87         }
  88
  89         return base + internal_offset;
  90 }
  91
  92 static unsigned sdma_v5_2_ring_init_cond_exec(struct amdgpu_ring *ring,
  93                                               uint64_t addr)
  94 {
  95         unsigned ret;
  96
  97         amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_COND_EXE));
  98         amdgpu_ring_write(ring, lower_32_bits(addr));
  99         amdgpu_ring_write(ring, upper_32_bits(addr));
 100         amdgpu_ring_write(ring, 1);
 101         /* this is the offset we need patch later */
 102         ret = ring->wptr & ring->buf_mask;
 103         /* insert dummy here and patch it later */
 104         amdgpu_ring_write(ring, 0);
 105
 106         return ret;
 107 }
 108
 109 /**
 110  * sdma_v5_2_ring_get_rptr - get the current read pointer
 111  *
 112  * @ring: amdgpu ring pointer
 113  *
 114  * Get the current rptr from the hardware (NAVI10+).
 115  */
 116 static uint64_t sdma_v5_2_ring_get_rptr(struct amdgpu_ring *ring)
 117 {
 118         u64 *rptr;
 119
 120         /* XXX check if swapping is necessary on BE */
 121         rptr = (u64 *)ring->rptr_cpu_addr;
 122
 123         DRM_DEBUG("rptr before shift == 0x%016llx\n", *rptr);
 124         return ((*rptr) >> 2);
 125 }
 126
 127 /**
 128  * sdma_v5_2_ring_get_wptr - get the current write pointer
 129  *
 130  * @ring: amdgpu ring pointer
 131  *
 132  * Get the current wptr from the hardware (NAVI10+).
 133  */
 134 static uint64_t sdma_v5_2_ring_get_wptr(struct amdgpu_ring *ring)
 135 {
 136         struct amdgpu_device *adev = ring->adev;
 137         u64 wptr;
 138
 139         if (ring->use_doorbell) {
 140                 /* XXX check if swapping is necessary on BE */
 141                 wptr = READ_ONCE(*((u64 *)ring->wptr_cpu_addr));
 142                 DRM_DEBUG("wptr/doorbell before shift == 0x%016llx\n", wptr);
 143         } else {
 144                 wptr = RREG32(sdma_v5_2_get_reg_offset(adev, ring->me, mmSDMA0_GFX_RB_WPTR_HI));
 145                 wptr = wptr << 32;
 146                 wptr |= RREG32(sdma_v5_2_get_reg_offset(adev, ring->me, mmSDMA0_GFX_RB_WPTR));
 147                 DRM_DEBUG("wptr before shift [%i] wptr == 0x%016llx\n", ring->me, wptr);
 148         }
 149
 150         return wptr >> 2;
 151 }
 152
 153 /**
 154  * sdma_v5_2_ring_set_wptr - commit the write pointer
 155  *
 156  * @ring: amdgpu ring pointer
 157  *
 158  * Write the wptr back to the hardware (NAVI10+).
 159  */
 160 static void sdma_v5_2_ring_set_wptr(struct amdgpu_ring *ring)
 161 {
 162         struct amdgpu_device *adev = ring->adev;
 163
 164         DRM_DEBUG("Setting write pointer\n");
 165         if (ring->use_doorbell) {
 166                 DRM_DEBUG("Using doorbell -- "
 167                                 "wptr_offs == 0x%08x "
 168                                 "lower_32_bits(ring->wptr << 2) == 0x%08x "
 169                                 "upper_32_bits(ring->wptr << 2) == 0x%08x\n",
 170                                 ring->wptr_offs,
 171                                 lower_32_bits(ring->wptr << 2),
 172                                 upper_32_bits(ring->wptr << 2));
 173                 /* XXX check if swapping is necessary on BE */
 174                 atomic64_set((atomic64_t *)ring->wptr_cpu_addr,
 175                              ring->wptr << 2);
 176                 DRM_DEBUG("calling WDOORBELL64(0x%08x, 0x%016llx)\n",
 177                                 ring->doorbell_index, ring->wptr << 2);
 178                 WDOORBELL64(ring->doorbell_index, ring->wptr << 2);
 179         } else {
 180                 DRM_DEBUG("Not using doorbell -- "
 181                                 "mmSDMA%i_GFX_RB_WPTR == 0x%08x "
 182                                 "mmSDMA%i_GFX_RB_WPTR_HI == 0x%08x\n",
 183                                 ring->me,
 184                                 lower_32_bits(ring->wptr << 2),
 185                                 ring->me,
 186                                 upper_32_bits(ring->wptr << 2));
 187                 WREG32(sdma_v5_2_get_reg_offset(adev, ring->me, mmSDMA0_GFX_RB_WPTR),
 188                         lower_32_bits(ring->wptr << 2));
 189                 WREG32(sdma_v5_2_get_reg_offset(adev, ring->me, mmSDMA0_GFX_RB_WPTR_HI),
 190                         upper_32_bits(ring->wptr << 2));
 191         }
 192 }
 193
 194 static void sdma_v5_2_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)
 195 {
 196         struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring);
 197         int i;
 198
 199         for (i = 0; i < count; i++)
 200                 if (sdma && sdma->burst_nop && (i == 0))
 201                         amdgpu_ring_write(ring, ring->funcs->nop |
 202                                 SDMA_PKT_NOP_HEADER_COUNT(count - 1));
 203                 else
 204                         amdgpu_ring_write(ring, ring->funcs->nop);
 205 }
 206
 207 /**
 208  * sdma_v5_2_ring_emit_ib - Schedule an IB on the DMA engine
 209  *
 210  * @ring: amdgpu ring pointer
 211  * @job: job to retrieve vmid from
 212  * @ib: IB object to schedule
 213  * @flags: unused
 214  *
 215  * Schedule an IB in the DMA ring.
 216  */
 217 static void sdma_v5_2_ring_emit_ib(struct amdgpu_ring *ring,
 218                                    struct amdgpu_job *job,
 219                                    struct amdgpu_ib *ib,
 220                                    uint32_t flags)
 221 {
 222         unsigned vmid = AMDGPU_JOB_GET_VMID(job);
 223         uint64_t csa_mc_addr = amdgpu_sdma_get_csa_mc_addr(ring, vmid);
 224
 225         /* An IB packet must end on a 8 DW boundary--the next dword
 226          * must be on a 8-dword boundary. Our IB packet below is 6
 227          * dwords long, thus add x number of NOPs, such that, in
 228          * modular arithmetic,
 229          * wptr + 6 + x = 8k, k >= 0, which in C is,
 230          * (wptr + 6 + x) % 8 = 0.
 231          * The expression below, is a solution of x.
 232          */
 233         sdma_v5_2_ring_insert_nop(ring, (2 - lower_32_bits(ring->wptr)) & 7);
 234
 235         amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_INDIRECT) |
 236                           SDMA_PKT_INDIRECT_HEADER_VMID(vmid & 0xf));
 237         /* base must be 32 byte aligned */
 238         amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr) & 0xffffffe0);
 239         amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
 240         amdgpu_ring_write(ring, ib->length_dw);
 241         amdgpu_ring_write(ring, lower_32_bits(csa_mc_addr));
 242         amdgpu_ring_write(ring, upper_32_bits(csa_mc_addr));
 243 }
 244
 245 /**
 246  * sdma_v5_2_ring_emit_mem_sync - flush the IB by graphics cache rinse
 247  *
 248  * @ring: amdgpu ring pointer
 249  *
 250  * flush the IB by graphics cache rinse.
 251  */
 252 static void sdma_v5_2_ring_emit_mem_sync(struct amdgpu_ring *ring)
 253 {
 254         uint32_t gcr_cntl = SDMA_GCR_GL2_INV | SDMA_GCR_GL2_WB |
 255                             SDMA_GCR_GLM_INV | SDMA_GCR_GL1_INV |
 256                             SDMA_GCR_GLV_INV | SDMA_GCR_GLK_INV |
 257                             SDMA_GCR_GLI_INV(1);
 258
 259         /* flush entire cache L0/L1/L2, this can be optimized by performance requirement */
 260         amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_GCR_REQ));
 261         amdgpu_ring_write(ring, SDMA_PKT_GCR_REQ_PAYLOAD1_BASE_VA_31_7(0));
 262         amdgpu_ring_write(ring, SDMA_PKT_GCR_REQ_PAYLOAD2_GCR_CONTROL_15_0(gcr_cntl) |
 263                         SDMA_PKT_GCR_REQ_PAYLOAD2_BASE_VA_47_32(0));
 264         amdgpu_ring_write(ring, SDMA_PKT_GCR_REQ_PAYLOAD3_LIMIT_VA_31_7(0) |
 265                         SDMA_PKT_GCR_REQ_PAYLOAD3_GCR_CONTROL_18_16(gcr_cntl >> 16));
 266         amdgpu_ring_write(ring, SDMA_PKT_GCR_REQ_PAYLOAD4_LIMIT_VA_47_32(0) |
 267                         SDMA_PKT_GCR_REQ_PAYLOAD4_VMID(0));
 268 }
 269
 270 /**
 271  * sdma_v5_2_ring_emit_hdp_flush - emit an hdp flush on the DMA ring
 272  *
 273  * @ring: amdgpu ring pointer
 274  *
 275  * Emit an hdp flush packet on the requested DMA ring.
 276  */
 277 static void sdma_v5_2_ring_emit_hdp_flush(struct amdgpu_ring *ring)
 278 {
 279         struct amdgpu_device *adev = ring->adev;
 280         u32 ref_and_mask = 0;
 281         const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg;
 282
 283         ref_and_mask = nbio_hf_reg->ref_and_mask_sdma0 << ring->me;
 284
 285         amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_POLL_REGMEM) |
 286                           SDMA_PKT_POLL_REGMEM_HEADER_HDP_FLUSH(1) |
 287                           SDMA_PKT_POLL_REGMEM_HEADER_FUNC(3)); /* == */
 288         amdgpu_ring_write(ring, (adev->nbio.funcs->get_hdp_flush_done_offset(adev)) << 2);
 289         amdgpu_ring_write(ring, (adev->nbio.funcs->get_hdp_flush_req_offset(adev)) << 2);
 290         amdgpu_ring_write(ring, ref_and_mask); /* reference */
 291         amdgpu_ring_write(ring, ref_and_mask); /* mask */
 292         amdgpu_ring_write(ring, SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff) |
 293                           SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(10)); /* retry count, poll interval */
 294 }
 295
 296 /**
 297  * sdma_v5_2_ring_emit_fence - emit a fence on the DMA ring
 298  *
 299  * @ring: amdgpu ring pointer
 300  * @addr: address
 301  * @seq: sequence number
 302  * @flags: fence related flags
 303  *
 304  * Add a DMA fence packet to the ring to write
 305  * the fence seq number and DMA trap packet to generate
 306  * an interrupt if needed.
 307  */
 308 static void sdma_v5_2_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq,
 309                                       unsigned flags)
 310 {
 311         bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
 312         /* write the fence */
 313         amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_FENCE) |
 314                           SDMA_PKT_FENCE_HEADER_MTYPE(0x3)); /* Ucached(UC) */
 315         /* zero in first two bits */
 316         BUG_ON(addr & 0x3);
 317         amdgpu_ring_write(ring, lower_32_bits(addr));
 318         amdgpu_ring_write(ring, upper_32_bits(addr));
 319         amdgpu_ring_write(ring, lower_32_bits(seq));
 320
 321         /* optionally write high bits as well */
 322         if (write64bit) {
 323                 addr += 4;
 324                 amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_FENCE) |
 325                                   SDMA_PKT_FENCE_HEADER_MTYPE(0x3));
 326                 /* zero in first two bits */
 327                 BUG_ON(addr & 0x3);
 328                 amdgpu_ring_write(ring, lower_32_bits(addr));
 329                 amdgpu_ring_write(ring, upper_32_bits(addr));
 330                 amdgpu_ring_write(ring, upper_32_bits(seq));
 331         }
 332
 333         if ((flags & AMDGPU_FENCE_FLAG_INT)) {
 334                 uint32_t ctx = ring->is_mes_queue ?
 335                         (ring->hw_queue_id | AMDGPU_FENCE_MES_QUEUE_FLAG) : 0;
 336                 /* generate an interrupt */
 337                 amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_TRAP));
 338                 amdgpu_ring_write(ring, SDMA_PKT_TRAP_INT_CONTEXT_INT_CONTEXT(ctx));
 339         }
 340 }
 341
 342
 343 /**
 344  * sdma_v5_2_gfx_stop - stop the gfx async dma engines
 345  *
 346  * @adev: amdgpu_device pointer
 347  *
 348  * Stop the gfx async dma ring buffers.
 349  */
 350 static void sdma_v5_2_gfx_stop(struct amdgpu_device *adev)
 351 {
 352         u32 rb_cntl, ib_cntl;
 353         int i;
 354
 355         for (i = 0; i < adev->sdma.num_instances; i++) {
 356                 rb_cntl = RREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_CNTL));
 357                 rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL, RB_ENABLE, 0);
 358                 WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_CNTL), rb_cntl);
 359                 ib_cntl = RREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_IB_CNTL));
 360                 ib_cntl = REG_SET_FIELD(ib_cntl, SDMA0_GFX_IB_CNTL, IB_ENABLE, 0);
 361                 WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_IB_CNTL), ib_cntl);
 362         }
 363 }
 364
 365 /**
 366  * sdma_v5_2_rlc_stop - stop the compute async dma engines
 367  *
 368  * @adev: amdgpu_device pointer
 369  *
 370  * Stop the compute async dma queues.
 371  */
 372 static void sdma_v5_2_rlc_stop(struct amdgpu_device *adev)
 373 {
 374         /* XXX todo */
 375 }
 376
 377 /**
 378  * sdma_v5_2_ctx_switch_enable - stop the async dma engines context switch
 379  *
 380  * @adev: amdgpu_device pointer
 381  * @enable: enable/disable the DMA MEs context switch.
 382  *
 383  * Halt or unhalt the async dma engines context switch.
 384  */
 385 static void sdma_v5_2_ctx_switch_enable(struct amdgpu_device *adev, bool enable)
 386 {
 387         u32 f32_cntl, phase_quantum = 0;
 388         int i;
 389
 390         if (amdgpu_sdma_phase_quantum) {
 391                 unsigned value = amdgpu_sdma_phase_quantum;
 392                 unsigned unit = 0;
 393
 394                 while (value > (SDMA0_PHASE0_QUANTUM__VALUE_MASK >>
 395                                 SDMA0_PHASE0_QUANTUM__VALUE__SHIFT)) {
 396                         value = (value + 1) >> 1;
 397                         unit++;
 398                 }
 399                 if (unit > (SDMA0_PHASE0_QUANTUM__UNIT_MASK >>
 400                             SDMA0_PHASE0_QUANTUM__UNIT__SHIFT)) {
 401                         value = (SDMA0_PHASE0_QUANTUM__VALUE_MASK >>
 402                                  SDMA0_PHASE0_QUANTUM__VALUE__SHIFT);
 403                         unit = (SDMA0_PHASE0_QUANTUM__UNIT_MASK >>
 404                                 SDMA0_PHASE0_QUANTUM__UNIT__SHIFT);
 405                         WARN_ONCE(1,
 406                         "clamping sdma_phase_quantum to %uK clock cycles\n",
 407                                   value << unit);
 408                 }
 409                 phase_quantum =
 410                         value << SDMA0_PHASE0_QUANTUM__VALUE__SHIFT |
 411                         unit  << SDMA0_PHASE0_QUANTUM__UNIT__SHIFT;
 412         }
 413
 414         for (i = 0; i < adev->sdma.num_instances; i++) {
 415                 if (enable && amdgpu_sdma_phase_quantum) {
 416                         WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_PHASE0_QUANTUM),
 417                                phase_quantum);
 418                         WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_PHASE1_QUANTUM),
 419                                phase_quantum);
 420                         WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_PHASE2_QUANTUM),
 421                                phase_quantum);
 422                 }
 423
 424                 if (!amdgpu_sriov_vf(adev)) {
 425                         f32_cntl = RREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_CNTL));
 426                         f32_cntl = REG_SET_FIELD(f32_cntl, SDMA0_CNTL,
 427                                         AUTO_CTXSW_ENABLE, enable ? 1 : 0);
 428                         WREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_CNTL), f32_cntl);
 429                 }
 430         }
 431
 432 }
 433
 434 /**
 435  * sdma_v5_2_enable - stop the async dma engines
 436  *
 437  * @adev: amdgpu_device pointer
 438  * @enable: enable/disable the DMA MEs.
 439  *
 440  * Halt or unhalt the async dma engines.
 441  */
 442 static void sdma_v5_2_enable(struct amdgpu_device *adev, bool enable)
 443 {
 444         u32 f32_cntl;
 445         int i;
 446
 447         if (!enable) {
 448                 sdma_v5_2_gfx_stop(adev);
 449                 sdma_v5_2_rlc_stop(adev);
 450         }
 451
 452         if (!amdgpu_sriov_vf(adev)) {
 453                 for (i = 0; i < adev->sdma.num_instances; i++) {
 454                         f32_cntl = RREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_F32_CNTL));
 455                         f32_cntl = REG_SET_FIELD(f32_cntl, SDMA0_F32_CNTL, HALT, enable ? 0 : 1);
 456                         WREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_F32_CNTL), f32_cntl);
 457                 }
 458         }
 459 }
 460
 461 /**
 462  * sdma_v5_2_gfx_resume - setup and start the async dma engines
 463  *
 464  * @adev: amdgpu_device pointer
 465  *
 466  * Set up the gfx DMA ring buffers and enable them.
 467  * Returns 0 for success, error for failure.
 468  */
 469 static int sdma_v5_2_gfx_resume(struct amdgpu_device *adev)
 470 {
 471         struct amdgpu_ring *ring;
 472         u32 rb_cntl, ib_cntl;
 473         u32 rb_bufsz;
 474         u32 doorbell;
 475         u32 doorbell_offset;
 476         u32 temp;
 477         u32 wptr_poll_cntl;
 478         u64 wptr_gpu_addr;
 479         int i, r;
 480
 481         for (i = 0; i < adev->sdma.num_instances; i++) {
 482                 ring = &adev->sdma.instance[i].ring;
 483
 484                 if (!amdgpu_sriov_vf(adev))
 485                         WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_SEM_WAIT_FAIL_TIMER_CNTL), 0);
 486
 487                 /* Set ring buffer size in dwords */
 488                 rb_bufsz = order_base_2(ring->ring_size / 4);
 489                 rb_cntl = RREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_CNTL));
 490                 rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL, RB_SIZE, rb_bufsz);
 491 #ifdef __BIG_ENDIAN
 492                 rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL, RB_SWAP_ENABLE, 1);
 493                 rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL,
 494                                         RPTR_WRITEBACK_SWAP_ENABLE, 1);
 495 #endif
 496                 WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_CNTL), rb_cntl);
 497
 498                 /* Initialize the ring buffer's read and write pointers */
 499                 WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_RPTR), 0);
 500                 WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_RPTR_HI), 0);
 501                 WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_WPTR), 0);
 502                 WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_WPTR_HI), 0);
 503
 504                 /* setup the wptr shadow polling */
 505                 wptr_gpu_addr = ring->wptr_gpu_addr;
 506                 WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_WPTR_POLL_ADDR_LO),
 507                        lower_32_bits(wptr_gpu_addr));
 508                 WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_WPTR_POLL_ADDR_HI),
 509                        upper_32_bits(wptr_gpu_addr));
 510                 wptr_poll_cntl = RREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i,
 511                                                          mmSDMA0_GFX_RB_WPTR_POLL_CNTL));
 512                 wptr_poll_cntl = REG_SET_FIELD(wptr_poll_cntl,
 513                                                SDMA0_GFX_RB_WPTR_POLL_CNTL,
 514                                                F32_POLL_ENABLE, 1);
 515                 WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_WPTR_POLL_CNTL),
 516                        wptr_poll_cntl);
 517
 518                 /* set the wb address whether it's enabled or not */
 519                 WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_RPTR_ADDR_HI),
 520                        upper_32_bits(ring->rptr_gpu_addr) & 0xFFFFFFFF);
 521                 WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_RPTR_ADDR_LO),
 522                        lower_32_bits(ring->rptr_gpu_addr) & 0xFFFFFFFC);
 523
 524                 rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL, RPTR_WRITEBACK_ENABLE, 1);
 525
 526                 WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_BASE), ring->gpu_addr >> 8);
 527                 WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_BASE_HI), ring->gpu_addr >> 40);
 528
 529                 ring->wptr = 0;
 530
 531                 /* before programing wptr to a less value, need set minor_ptr_update first */
 532                 WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_MINOR_PTR_UPDATE), 1);
 533
 534                 if (!amdgpu_sriov_vf(adev)) { /* only bare-metal use register write for wptr */
 535                         WREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_WPTR), lower_32_bits(ring->wptr << 2));
 536                         WREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_WPTR_HI), upper_32_bits(ring->wptr << 2));
 537                 }
 538
 539                 doorbell = RREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_DOORBELL));
 540                 doorbell_offset = RREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_DOORBELL_OFFSET));
 541
 542                 if (ring->use_doorbell) {
 543                         doorbell = REG_SET_FIELD(doorbell, SDMA0_GFX_DOORBELL, ENABLE, 1);
 544                         doorbell_offset = REG_SET_FIELD(doorbell_offset, SDMA0_GFX_DOORBELL_OFFSET,
 545                                         OFFSET, ring->doorbell_index);
 546                 } else {
 547                         doorbell = REG_SET_FIELD(doorbell, SDMA0_GFX_DOORBELL, ENABLE, 0);
 548                 }
 549                 WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_DOORBELL), doorbell);
 550                 WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_DOORBELL_OFFSET), doorbell_offset);
 551
 552                 adev->nbio.funcs->sdma_doorbell_range(adev, i, ring->use_doorbell,
 553                                                       ring->doorbell_index,
 554                                                       adev->doorbell_index.sdma_doorbell_range);
 555
 556                 if (amdgpu_sriov_vf(adev))
 557                         sdma_v5_2_ring_set_wptr(ring);
 558
 559                 /* set minor_ptr_update to 0 after wptr programed */
 560
 561                 WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_MINOR_PTR_UPDATE), 0);
 562
 563                 /* SRIOV VF has no control of any of registers below */
 564                 if (!amdgpu_sriov_vf(adev)) {
 565                         /* set utc l1 enable flag always to 1 */
 566                         temp = RREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_CNTL));
 567                         temp = REG_SET_FIELD(temp, SDMA0_CNTL, UTC_L1_ENABLE, 1);
 568
 569                         /* enable MCBP */
 570                         temp = REG_SET_FIELD(temp, SDMA0_CNTL, MIDCMD_PREEMPT_ENABLE, 1);
 571                         WREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_CNTL), temp);
 572
 573                         /* Set up RESP_MODE to non-copy addresses */
 574                         temp = RREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_UTCL1_CNTL));
 575                         temp = REG_SET_FIELD(temp, SDMA0_UTCL1_CNTL, RESP_MODE, 3);
 576                         temp = REG_SET_FIELD(temp, SDMA0_UTCL1_CNTL, REDO_DELAY, 9);
 577                         WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_UTCL1_CNTL), temp);
 578
 579                         /* program default cache read and write policy */
 580                         temp = RREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_UTCL1_PAGE));
 581                         /* clean read policy and write policy bits */
 582                         temp &= 0xFF0FFF;
 583                         temp |= ((CACHE_READ_POLICY_L2__DEFAULT << 12) |
 584                                  (CACHE_WRITE_POLICY_L2__DEFAULT << 14) |
 585                                  SDMA0_UTCL1_PAGE__LLC_NOALLOC_MASK);
 586                         WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_UTCL1_PAGE), temp);
 587
 588                         /* unhalt engine */
 589                         temp = RREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_F32_CNTL));
 590                         temp = REG_SET_FIELD(temp, SDMA0_F32_CNTL, HALT, 0);
 591                         WREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_F32_CNTL), temp);
 592                 }
 593
 594                 /* enable DMA RB */
 595                 rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL, RB_ENABLE, 1);
 596                 WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_CNTL), rb_cntl);
 597
 598                 ib_cntl = RREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_IB_CNTL));
 599                 ib_cntl = REG_SET_FIELD(ib_cntl, SDMA0_GFX_IB_CNTL, IB_ENABLE, 1);
 600 #ifdef __BIG_ENDIAN
 601                 ib_cntl = REG_SET_FIELD(ib_cntl, SDMA0_GFX_IB_CNTL, IB_SWAP_ENABLE, 1);
 602 #endif
 603                 /* enable DMA IBs */
 604                 WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_IB_CNTL), ib_cntl);
 605
 606                 if (amdgpu_sriov_vf(adev)) { /* bare-metal sequence doesn't need below to lines */
 607                         sdma_v5_2_ctx_switch_enable(adev, true);
 608                         sdma_v5_2_enable(adev, true);
 609                 }
 610
 611                 r = amdgpu_ring_test_helper(ring);
 612                 if (r)
 613                         return r;
 614         }
 615
 616         return 0;
 617 }
 618
 619 /**
 620  * sdma_v5_2_rlc_resume - setup and start the async dma engines
 621  *
 622  * @adev: amdgpu_device pointer
 623  *
 624  * Set up the compute DMA queues and enable them.
 625  * Returns 0 for success, error for failure.
 626  */
 627 static int sdma_v5_2_rlc_resume(struct amdgpu_device *adev)
 628 {
 629         return 0;
 630 }
 631
 632 /**
 633  * sdma_v5_2_load_microcode - load the sDMA ME ucode
 634  *
 635  * @adev: amdgpu_device pointer
 636  *
 637  * Loads the sDMA0/1/2/3 ucode.
 638  * Returns 0 for success, -EINVAL if the ucode is not available.
 639  */
 640 static int sdma_v5_2_load_microcode(struct amdgpu_device *adev)
 641 {
 642         const struct sdma_firmware_header_v1_0 *hdr;
 643         const __le32 *fw_data;
 644         u32 fw_size;
 645         int i, j;
 646
 647         /* halt the MEs */
 648         sdma_v5_2_enable(adev, false);
 649
 650         for (i = 0; i < adev->sdma.num_instances; i++) {
 651                 if (!adev->sdma.instance[i].fw)
 652                         return -EINVAL;
 653
 654                 hdr = (const struct sdma_firmware_header_v1_0 *)adev->sdma.instance[i].fw->data;
 655                 amdgpu_ucode_print_sdma_hdr(&hdr->header);
 656                 fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
 657
 658                 fw_data = (const __le32 *)
 659                         (adev->sdma.instance[i].fw->data +
 660                                 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
 661
 662                 WREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_UCODE_ADDR), 0);
 663
 664                 for (j = 0; j < fw_size; j++) {
 665                         if (amdgpu_emu_mode == 1 && j % 500 == 0)
 666                                 msleep(1);
 667                         WREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_UCODE_DATA), le32_to_cpup(fw_data++));
 668                 }
 669
 670                 WREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_UCODE_ADDR), adev->sdma.instance[i].fw_version);
 671         }
 672
 673         return 0;
 674 }
 675
 676 static int sdma_v5_2_soft_reset(void *handle)
 677 {
 678         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 679         u32 grbm_soft_reset;
 680         u32 tmp;
 681         int i;
 682
 683         for (i = 0; i < adev->sdma.num_instances; i++) {
 684                 grbm_soft_reset = REG_SET_FIELD(0,
 685                                                 GRBM_SOFT_RESET, SOFT_RESET_SDMA0,
 686                                                 1);
 687                 grbm_soft_reset <<= i;
 688
 689                 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
 690                 tmp |= grbm_soft_reset;
 691                 DRM_DEBUG("GRBM_SOFT_RESET=0x%08X\n", tmp);
 692                 WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp);
 693                 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
 694
 695                 udelay(50);
 696
 697                 tmp &= ~grbm_soft_reset;
 698                 WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp);
 699                 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
 700
 701                 udelay(50);
 702         }
 703
 704         return 0;
 705 }
 706
 707 /**
 708  * sdma_v5_2_start - setup and start the async dma engines
 709  *
 710  * @adev: amdgpu_device pointer
 711  *
 712  * Set up the DMA engines and enable them.
 713  * Returns 0 for success, error for failure.
 714  */
 715 static int sdma_v5_2_start(struct amdgpu_device *adev)
 716 {
 717         int r = 0;
 718
 719         if (amdgpu_sriov_vf(adev)) {
 720                 sdma_v5_2_ctx_switch_enable(adev, false);
 721                 sdma_v5_2_enable(adev, false);
 722
 723                 /* set RB registers */
 724                 r = sdma_v5_2_gfx_resume(adev);
 725                 return r;
 726         }
 727
 728         if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
 729                 r = sdma_v5_2_load_microcode(adev);
 730                 if (r)
 731                         return r;
 732
 733                 /* The value of mmSDMA_F32_CNTL is invalid the moment after loading fw */
 734                 if (amdgpu_emu_mode == 1)
 735                         msleep(1000);
 736         }
 737
 738         sdma_v5_2_soft_reset(adev);
 739         /* unhalt the MEs */
 740         sdma_v5_2_enable(adev, true);
 741         /* enable sdma ring preemption */
 742         sdma_v5_2_ctx_switch_enable(adev, true);
 743
 744         /* start the gfx rings and rlc compute queues */
 745         r = sdma_v5_2_gfx_resume(adev);
 746         if (r)
 747                 return r;
 748         r = sdma_v5_2_rlc_resume(adev);
 749
 750         return r;
 751 }
 752
 753 static int sdma_v5_2_mqd_init(struct amdgpu_device *adev, void *mqd,
 754                               struct amdgpu_mqd_prop *prop)
 755 {
 756         struct v10_sdma_mqd *m = mqd;
 757         uint64_t wb_gpu_addr;
 758
 759         m->sdmax_rlcx_rb_cntl =
 760                 order_base_2(prop->queue_size / 4) << SDMA0_RLC0_RB_CNTL__RB_SIZE__SHIFT |
 761                 1 << SDMA0_RLC0_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT |
 762                 6 << SDMA0_RLC0_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT |
 763                 1 << SDMA0_RLC0_RB_CNTL__RB_PRIV__SHIFT;
 764
 765         m->sdmax_rlcx_rb_base = lower_32_bits(prop->hqd_base_gpu_addr >> 8);
 766         m->sdmax_rlcx_rb_base_hi = upper_32_bits(prop->hqd_base_gpu_addr >> 8);
 767
 768         m->sdmax_rlcx_rb_wptr_poll_cntl = RREG32(sdma_v5_2_get_reg_offset(adev, 0,
 769                                                   mmSDMA0_GFX_RB_WPTR_POLL_CNTL));
 770
 771         wb_gpu_addr = prop->wptr_gpu_addr;
 772         m->sdmax_rlcx_rb_wptr_poll_addr_lo = lower_32_bits(wb_gpu_addr);
 773         m->sdmax_rlcx_rb_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr);
 774
 775         wb_gpu_addr = prop->rptr_gpu_addr;
 776         m->sdmax_rlcx_rb_rptr_addr_lo = lower_32_bits(wb_gpu_addr);
 777         m->sdmax_rlcx_rb_rptr_addr_hi = upper_32_bits(wb_gpu_addr);
 778
 779         m->sdmax_rlcx_ib_cntl = RREG32(sdma_v5_2_get_reg_offset(adev, 0,
 780                                                         mmSDMA0_GFX_IB_CNTL));
 781
 782         m->sdmax_rlcx_doorbell_offset =
 783                 prop->doorbell_index << SDMA0_RLC0_DOORBELL_OFFSET__OFFSET__SHIFT;
 784
 785         m->sdmax_rlcx_doorbell = REG_SET_FIELD(0, SDMA0_RLC0_DOORBELL, ENABLE, 1);
 786
 787         return 0;
 788 }
 789
 790 static void sdma_v5_2_set_mqd_funcs(struct amdgpu_device *adev)
 791 {
 792         adev->mqds[AMDGPU_HW_IP_DMA].mqd_size = sizeof(struct v10_sdma_mqd);
 793         adev->mqds[AMDGPU_HW_IP_DMA].init_mqd = sdma_v5_2_mqd_init;
 794 }
 795
 796 /**
 797  * sdma_v5_2_ring_test_ring - simple async dma engine test
 798  *
 799  * @ring: amdgpu_ring structure holding ring information
 800  *
 801  * Test the DMA engine by writing using it to write an
 802  * value to memory.
 803  * Returns 0 for success, error for failure.
 804  */
 805 static int sdma_v5_2_ring_test_ring(struct amdgpu_ring *ring)
 806 {
 807         struct amdgpu_device *adev = ring->adev;
 808         unsigned i;
 809         unsigned index;
 810         int r;
 811         u32 tmp;
 812         u64 gpu_addr;
 813         volatile uint32_t *cpu_ptr = NULL;
 814
 815         tmp = 0xCAFEDEAD;
 816
 817         if (ring->is_mes_queue) {
 818                 uint32_t offset = 0;
 819                 offset = amdgpu_mes_ctx_get_offs(ring,
 820                                          AMDGPU_MES_CTX_PADDING_OFFS);
 821                 gpu_addr = amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset);
 822                 cpu_ptr = amdgpu_mes_ctx_get_offs_cpu_addr(ring, offset);
 823                 *cpu_ptr = tmp;
 824         } else {
 825                 r = amdgpu_device_wb_get(adev, &index);
 826                 if (r) {
 827                         dev_err(adev->dev, "(%d) failed to allocate wb slot\n", r);
 828                         return r;
 829                 }
 830
 831                 gpu_addr = adev->wb.gpu_addr + (index * 4);
 832                 adev->wb.wb[index] = cpu_to_le32(tmp);
 833         }
 834
 835         r = amdgpu_ring_alloc(ring, 20);
 836         if (r) {
 837                 DRM_ERROR("amdgpu: dma failed to lock ring %d (%d).\n", ring->idx, r);
 838                 amdgpu_device_wb_free(adev, index);
 839                 return r;
 840         }
 841
 842         amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) |
 843                           SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR));
 844         amdgpu_ring_write(ring, lower_32_bits(gpu_addr));
 845         amdgpu_ring_write(ring, upper_32_bits(gpu_addr));
 846         amdgpu_ring_write(ring, SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(0));
 847         amdgpu_ring_write(ring, 0xDEADBEEF);
 848         amdgpu_ring_commit(ring);
 849
 850         for (i = 0; i < adev->usec_timeout; i++) {
 851                 if (ring->is_mes_queue)
 852                         tmp = le32_to_cpu(*cpu_ptr);
 853                 else
 854                         tmp = le32_to_cpu(adev->wb.wb[index]);
 855                 if (tmp == 0xDEADBEEF)
 856                         break;
 857                 if (amdgpu_emu_mode == 1)
 858                         msleep(1);
 859                 else
 860                         udelay(1);
 861         }
 862
 863         if (i >= adev->usec_timeout)
 864                 r = -ETIMEDOUT;
 865
 866         if (!ring->is_mes_queue)
 867                 amdgpu_device_wb_free(adev, index);
 868
 869         return r;
 870 }
 871
 872 /**
 873  * sdma_v5_2_ring_test_ib - test an IB on the DMA engine
 874  *
 875  * @ring: amdgpu_ring structure holding ring information
 876  * @timeout: timeout value in jiffies, or MAX_SCHEDULE_TIMEOUT
 877  *
 878  * Test a simple IB in the DMA ring.
 879  * Returns 0 on success, error on failure.
 880  */
 881 static int sdma_v5_2_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 882 {
 883         struct amdgpu_device *adev = ring->adev;
 884         struct amdgpu_ib ib;
 885         struct dma_fence *f = NULL;
 886         unsigned index;
 887         long r;
 888         u32 tmp = 0;
 889         u64 gpu_addr;
 890         volatile uint32_t *cpu_ptr = NULL;
 891
 892         tmp = 0xCAFEDEAD;
 893         memset(&ib, 0, sizeof(ib));
 894
 895         if (ring->is_mes_queue) {
 896                 uint32_t offset = 0;
 897                 offset = amdgpu_mes_ctx_get_offs(ring, AMDGPU_MES_CTX_IB_OFFS);
 898                 ib.gpu_addr = amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset);
 899                 ib.ptr = (void *)amdgpu_mes_ctx_get_offs_cpu_addr(ring, offset);
 900
 901                 offset = amdgpu_mes_ctx_get_offs(ring,
 902                                          AMDGPU_MES_CTX_PADDING_OFFS);
 903                 gpu_addr = amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset);
 904                 cpu_ptr = amdgpu_mes_ctx_get_offs_cpu_addr(ring, offset);
 905                 *cpu_ptr = tmp;
 906         } else {
 907                 r = amdgpu_device_wb_get(adev, &index);
 908                 if (r) {
 909                         dev_err(adev->dev, "(%ld) failed to allocate wb slot\n", r);
 910                         return r;
 911                 }
 912
 913                 gpu_addr = adev->wb.gpu_addr + (index * 4);
 914                 adev->wb.wb[index] = cpu_to_le32(tmp);
 915
 916                 r = amdgpu_ib_get(adev, NULL, 256, AMDGPU_IB_POOL_DIRECT, &ib);
 917                 if (r) {
 918                         DRM_ERROR("amdgpu: failed to get ib (%ld).\n", r);
 919                         goto err0;
 920                 }
 921         }
 922
 923         ib.ptr[0] = SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) |
 924                 SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR);
 925         ib.ptr[1] = lower_32_bits(gpu_addr);
 926         ib.ptr[2] = upper_32_bits(gpu_addr);
 927         ib.ptr[3] = SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(0);
 928         ib.ptr[4] = 0xDEADBEEF;
 929         ib.ptr[5] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
 930         ib.ptr[6] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
 931         ib.ptr[7] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
 932         ib.length_dw = 8;
 933
 934         r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
 935         if (r)
 936                 goto err1;
 937
 938         r = dma_fence_wait_timeout(f, false, timeout);
 939         if (r == 0) {
 940                 DRM_ERROR("amdgpu: IB test timed out\n");
 941                 r = -ETIMEDOUT;
 942                 goto err1;
 943         } else if (r < 0) {
 944                 DRM_ERROR("amdgpu: fence wait failed (%ld).\n", r);
 945                 goto err1;
 946         }
 947
 948         if (ring->is_mes_queue)
 949                 tmp = le32_to_cpu(*cpu_ptr);
 950         else
 951                 tmp = le32_to_cpu(adev->wb.wb[index]);
 952
 953         if (tmp == 0xDEADBEEF)
 954                 r = 0;
 955         else
 956                 r = -EINVAL;
 957
 958 err1:
 959         amdgpu_ib_free(adev, &ib, NULL);
 960         dma_fence_put(f);
 961 err0:
 962         if (!ring->is_mes_queue)
 963                 amdgpu_device_wb_free(adev, index);
 964         return r;
 965 }
 966
 967
 968 /**
 969  * sdma_v5_2_vm_copy_pte - update PTEs by copying them from the GART
 970  *
 971  * @ib: indirect buffer to fill with commands
 972  * @pe: addr of the page entry
 973  * @src: src addr to copy from
 974  * @count: number of page entries to update
 975  *
 976  * Update PTEs by copying them from the GART using sDMA.
 977  */
 978 static void sdma_v5_2_vm_copy_pte(struct amdgpu_ib *ib,
 979                                   uint64_t pe, uint64_t src,
 980                                   unsigned count)
 981 {
 982         unsigned bytes = count * 8;
 983
 984         ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_COPY) |
 985                 SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR);
 986         ib->ptr[ib->length_dw++] = bytes - 1;
 987         ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */
 988         ib->ptr[ib->length_dw++] = lower_32_bits(src);
 989         ib->ptr[ib->length_dw++] = upper_32_bits(src);
 990         ib->ptr[ib->length_dw++] = lower_32_bits(pe);
 991         ib->ptr[ib->length_dw++] = upper_32_bits(pe);
 992
 993 }
 994
 995 /**
 996  * sdma_v5_2_vm_write_pte - update PTEs by writing them manually
 997  *
 998  * @ib: indirect buffer to fill with commands
 999  * @pe: addr of the page entry
1000  * @value: dst addr to write into pe
1001  * @count: number of page entries to update
1002  * @incr: increase next addr by incr bytes
1003  *
1004  * Update PTEs by writing them manually using sDMA.
1005  */
1006 static void sdma_v5_2_vm_write_pte(struct amdgpu_ib *ib, uint64_t pe,
1007                                    uint64_t value, unsigned count,
1008                                    uint32_t incr)
1009 {
1010         unsigned ndw = count * 2;
1011
1012         ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) |
1013                 SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR);
1014         ib->ptr[ib->length_dw++] = lower_32_bits(pe);
1015         ib->ptr[ib->length_dw++] = upper_32_bits(pe);
1016         ib->ptr[ib->length_dw++] = ndw - 1;
1017         for (; ndw > 0; ndw -= 2) {
1018                 ib->ptr[ib->length_dw++] = lower_32_bits(value);
1019                 ib->ptr[ib->length_dw++] = upper_32_bits(value);
1020                 value += incr;
1021         }
1022 }
1023
1024 /**
1025  * sdma_v5_2_vm_set_pte_pde - update the page tables using sDMA
1026  *
1027  * @ib: indirect buffer to fill with commands
1028  * @pe: addr of the page entry
1029  * @addr: dst addr to write into pe
1030  * @count: number of page entries to update
1031  * @incr: increase next addr by incr bytes
1032  * @flags: access flags
1033  *
1034  * Update the page tables using sDMA.
1035  */
1036 static void sdma_v5_2_vm_set_pte_pde(struct amdgpu_ib *ib,
1037                                      uint64_t pe,
1038                                      uint64_t addr, unsigned count,
1039                                      uint32_t incr, uint64_t flags)
1040 {
1041         /* for physically contiguous pages (vram) */
1042         ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_PTEPDE);
1043         ib->ptr[ib->length_dw++] = lower_32_bits(pe); /* dst addr */
1044         ib->ptr[ib->length_dw++] = upper_32_bits(pe);
1045         ib->ptr[ib->length_dw++] = lower_32_bits(flags); /* mask */
1046         ib->ptr[ib->length_dw++] = upper_32_bits(flags);
1047         ib->ptr[ib->length_dw++] = lower_32_bits(addr); /* value */
1048         ib->ptr[ib->length_dw++] = upper_32_bits(addr);
1049         ib->ptr[ib->length_dw++] = incr; /* increment size */
1050         ib->ptr[ib->length_dw++] = 0;
1051         ib->ptr[ib->length_dw++] = count - 1; /* number of entries */
1052 }
1053
1054 /**
1055  * sdma_v5_2_ring_pad_ib - pad the IB
1056  *
1057  * @ib: indirect buffer to fill with padding
1058  * @ring: amdgpu_ring structure holding ring information
1059  *
1060  * Pad the IB with NOPs to a boundary multiple of 8.
1061  */
1062 static void sdma_v5_2_ring_pad_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib)
1063 {
1064         struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring);
1065         u32 pad_count;
1066         int i;
1067
1068         pad_count = (-ib->length_dw) & 0x7;
1069         for (i = 0; i < pad_count; i++)
1070                 if (sdma && sdma->burst_nop && (i == 0))
1071                         ib->ptr[ib->length_dw++] =
1072                                 SDMA_PKT_HEADER_OP(SDMA_OP_NOP) |
1073                                 SDMA_PKT_NOP_HEADER_COUNT(pad_count - 1);
1074                 else
1075                         ib->ptr[ib->length_dw++] =
1076                                 SDMA_PKT_HEADER_OP(SDMA_OP_NOP);
1077 }
1078
1079
1080 /**
1081  * sdma_v5_2_ring_emit_pipeline_sync - sync the pipeline
1082  *
1083  * @ring: amdgpu_ring pointer
1084  *
1085  * Make sure all previous operations are completed (CIK).
1086  */
1087 static void sdma_v5_2_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
1088 {
1089         uint32_t seq = ring->fence_drv.sync_seq;
1090         uint64_t addr = ring->fence_drv.gpu_addr;
1091
1092         /* wait for idle */
1093         amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_POLL_REGMEM) |
1094                           SDMA_PKT_POLL_REGMEM_HEADER_HDP_FLUSH(0) |
1095                           SDMA_PKT_POLL_REGMEM_HEADER_FUNC(3) | /* equal */
1096                           SDMA_PKT_POLL_REGMEM_HEADER_MEM_POLL(1));
1097         amdgpu_ring_write(ring, addr & 0xfffffffc);
1098         amdgpu_ring_write(ring, upper_32_bits(addr) & 0xffffffff);
1099         amdgpu_ring_write(ring, seq); /* reference */
1100         amdgpu_ring_write(ring, 0xffffffff); /* mask */
1101         amdgpu_ring_write(ring, SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff) |
1102                           SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(4)); /* retry count, poll interval */
1103 }
1104
1105
1106 /**
1107  * sdma_v5_2_ring_emit_vm_flush - vm flush using sDMA
1108  *
1109  * @ring: amdgpu_ring pointer
1110  * @vmid: vmid number to use
1111  * @pd_addr: address
1112  *
1113  * Update the page table base and flush the VM TLB
1114  * using sDMA.
1115  */
1116 static void sdma_v5_2_ring_emit_vm_flush(struct amdgpu_ring *ring,
1117                                          unsigned vmid, uint64_t pd_addr)
1118 {
1119         amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
1120 }
1121
1122 static void sdma_v5_2_ring_emit_wreg(struct amdgpu_ring *ring,
1123                                      uint32_t reg, uint32_t val)
1124 {
1125         amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_SRBM_WRITE) |
1126                           SDMA_PKT_SRBM_WRITE_HEADER_BYTE_EN(0xf));
1127         amdgpu_ring_write(ring, reg);
1128         amdgpu_ring_write(ring, val);
1129 }
1130
1131 static void sdma_v5_2_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
1132                                          uint32_t val, uint32_t mask)
1133 {
1134         amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_POLL_REGMEM) |
1135                           SDMA_PKT_POLL_REGMEM_HEADER_HDP_FLUSH(0) |
1136                           SDMA_PKT_POLL_REGMEM_HEADER_FUNC(3)); /* equal */
1137         amdgpu_ring_write(ring, reg << 2);
1138         amdgpu_ring_write(ring, 0);
1139         amdgpu_ring_write(ring, val); /* reference */
1140         amdgpu_ring_write(ring, mask); /* mask */
1141         amdgpu_ring_write(ring, SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff) |
1142                           SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(10));
1143 }
1144
1145 static void sdma_v5_2_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring,
1146                                                    uint32_t reg0, uint32_t reg1,
1147                                                    uint32_t ref, uint32_t mask)
1148 {
1149         amdgpu_ring_emit_wreg(ring, reg0, ref);
1150         /* wait for a cycle to reset vm_inv_eng*_ack */
1151         amdgpu_ring_emit_reg_wait(ring, reg0, 0, 0);
1152         amdgpu_ring_emit_reg_wait(ring, reg1, mask, mask);
1153 }
1154
1155 static int sdma_v5_2_early_init(void *handle)
1156 {
1157         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1158         int r;
1159
1160         r = amdgpu_sdma_init_microcode(adev, 0, true);
1161         if (r)
1162                 return r;
1163
1164         sdma_v5_2_set_ring_funcs(adev);
1165         sdma_v5_2_set_buffer_funcs(adev);
1166         sdma_v5_2_set_vm_pte_funcs(adev);
1167         sdma_v5_2_set_irq_funcs(adev);
1168         sdma_v5_2_set_mqd_funcs(adev);
1169
1170         return 0;
1171 }
1172
1173 static unsigned sdma_v5_2_seq_to_irq_id(int seq_num)
1174 {
1175         switch (seq_num) {
1176         case 0:
1177                 return SOC15_IH_CLIENTID_SDMA0;
1178         case 1:
1179                 return SOC15_IH_CLIENTID_SDMA1;
1180         case 2:
1181                 return SOC15_IH_CLIENTID_SDMA2;
1182         case 3:
1183                 return SOC15_IH_CLIENTID_SDMA3_Sienna_Cichlid;
1184         default:
1185                 break;
1186         }
1187         return -EINVAL;
1188 }
1189
1190 static unsigned sdma_v5_2_seq_to_trap_id(int seq_num)
1191 {
1192         switch (seq_num) {
1193         case 0:
1194                 return SDMA0_5_0__SRCID__SDMA_TRAP;
1195         case 1:
1196                 return SDMA1_5_0__SRCID__SDMA_TRAP;
1197         case 2:
1198                 return SDMA2_5_0__SRCID__SDMA_TRAP;
1199         case 3:
1200                 return SDMA3_5_0__SRCID__SDMA_TRAP;
1201         default:
1202                 break;
1203         }
1204         return -EINVAL;
1205 }
1206
1207 static int sdma_v5_2_sw_init(void *handle)
1208 {
1209         struct amdgpu_ring *ring;
1210         int r, i;
1211         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1212
1213         /* SDMA trap event */
1214         for (i = 0; i < adev->sdma.num_instances; i++) {
1215                 r = amdgpu_irq_add_id(adev, sdma_v5_2_seq_to_irq_id(i),
1216                                       sdma_v5_2_seq_to_trap_id(i),
1217                                       &adev->sdma.trap_irq);
1218                 if (r)
1219                         return r;
1220         }
1221
1222         for (i = 0; i < adev->sdma.num_instances; i++) {
1223                 ring = &adev->sdma.instance[i].ring;
1224                 ring->ring_obj = NULL;
1225                 ring->use_doorbell = true;
1226                 ring->me = i;
1227
1228                 DRM_INFO("use_doorbell being set to: [%s]\n",
1229                                 ring->use_doorbell?"true":"false");
1230
1231                 ring->doorbell_index =
1232                         (adev->doorbell_index.sdma_engine[i] << 1); //get DWORD offset
1233
1234                 ring->vm_hub = AMDGPU_GFXHUB(0);
1235                 sprintf(ring->name, "sdma%d", i);
1236                 r = amdgpu_ring_init(adev, ring, 1024, &adev->sdma.trap_irq,
1237                                      AMDGPU_SDMA_IRQ_INSTANCE0 + i,
1238                                      AMDGPU_RING_PRIO_DEFAULT, NULL);
1239                 if (r)
1240                         return r;
1241         }
1242
1243         return r;
1244 }
1245
1246 static int sdma_v5_2_sw_fini(void *handle)
1247 {
1248         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1249         int i;
1250
1251         for (i = 0; i < adev->sdma.num_instances; i++)
1252                 amdgpu_ring_fini(&adev->sdma.instance[i].ring);
1253
1254         amdgpu_sdma_destroy_inst_ctx(adev, true);
1255
1256         return 0;
1257 }
1258
1259 static int sdma_v5_2_hw_init(void *handle)
1260 {
1261         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1262
1263         return sdma_v5_2_start(adev);
1264 }
1265
1266 static int sdma_v5_2_hw_fini(void *handle)
1267 {
1268         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1269
1270         if (amdgpu_sriov_vf(adev))
1271                 return 0;
1272
1273         sdma_v5_2_ctx_switch_enable(adev, false);
1274         sdma_v5_2_enable(adev, false);
1275
1276         return 0;
1277 }
1278
1279 static int sdma_v5_2_suspend(void *handle)
1280 {
1281         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1282
1283         return sdma_v5_2_hw_fini(adev);
1284 }
1285
1286 static int sdma_v5_2_resume(void *handle)
1287 {
1288         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1289
1290         return sdma_v5_2_hw_init(adev);
1291 }
1292
1293 static bool sdma_v5_2_is_idle(void *handle)
1294 {
1295         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1296         u32 i;
1297
1298         for (i = 0; i < adev->sdma.num_instances; i++) {
1299                 u32 tmp = RREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_STATUS_REG));
1300
1301                 if (!(tmp & SDMA0_STATUS_REG__IDLE_MASK))
1302                         return false;
1303         }
1304
1305         return true;
1306 }
1307
1308 static int sdma_v5_2_wait_for_idle(void *handle)
1309 {
1310         unsigned i;
1311         u32 sdma0, sdma1, sdma2, sdma3;
1312         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1313
1314         for (i = 0; i < adev->usec_timeout; i++) {
1315                 sdma0 = RREG32(sdma_v5_2_get_reg_offset(adev, 0, mmSDMA0_STATUS_REG));
1316                 sdma1 = RREG32(sdma_v5_2_get_reg_offset(adev, 1, mmSDMA0_STATUS_REG));
1317                 sdma2 = RREG32(sdma_v5_2_get_reg_offset(adev, 2, mmSDMA0_STATUS_REG));
1318                 sdma3 = RREG32(sdma_v5_2_get_reg_offset(adev, 3, mmSDMA0_STATUS_REG));
1319
1320                 if (sdma0 & sdma1 & sdma2 & sdma3 & SDMA0_STATUS_REG__IDLE_MASK)
1321                         return 0;
1322                 udelay(1);
1323         }
1324         return -ETIMEDOUT;
1325 }
1326
1327 static int sdma_v5_2_ring_preempt_ib(struct amdgpu_ring *ring)
1328 {
1329         int i, r = 0;
1330         struct amdgpu_device *adev = ring->adev;
1331         u32 index = 0;
1332         u64 sdma_gfx_preempt;
1333
1334         amdgpu_sdma_get_index_from_ring(ring, &index);
1335         sdma_gfx_preempt =
1336                 sdma_v5_2_get_reg_offset(adev, index, mmSDMA0_GFX_PREEMPT);
1337
1338         /* assert preemption condition */
1339         amdgpu_ring_set_preempt_cond_exec(ring, false);
1340
1341         /* emit the trailing fence */
1342         ring->trail_seq += 1;
1343         amdgpu_ring_alloc(ring, 10);
1344         sdma_v5_2_ring_emit_fence(ring, ring->trail_fence_gpu_addr,
1345                                   ring->trail_seq, 0);
1346         amdgpu_ring_commit(ring);
1347
1348         /* assert IB preemption */
1349         WREG32(sdma_gfx_preempt, 1);
1350
1351         /* poll the trailing fence */
1352         for (i = 0; i < adev->usec_timeout; i++) {
1353                 if (ring->trail_seq ==
1354                     le32_to_cpu(*(ring->trail_fence_cpu_addr)))
1355                         break;
1356                 udelay(1);
1357         }
1358
1359         if (i >= adev->usec_timeout) {
1360                 r = -EINVAL;
1361                 DRM_ERROR("ring %d failed to be preempted\n", ring->idx);
1362         }
1363
1364         /* deassert IB preemption */
1365         WREG32(sdma_gfx_preempt, 0);
1366
1367         /* deassert the preemption condition */
1368         amdgpu_ring_set_preempt_cond_exec(ring, true);
1369         return r;
1370 }
1371
1372 static int sdma_v5_2_set_trap_irq_state(struct amdgpu_device *adev,
1373                                         struct amdgpu_irq_src *source,
1374                                         unsigned type,
1375                                         enum amdgpu_interrupt_state state)
1376 {
1377         u32 sdma_cntl;
1378         u32 reg_offset = sdma_v5_2_get_reg_offset(adev, type, mmSDMA0_CNTL);
1379
1380         if (!amdgpu_sriov_vf(adev)) {
1381                 sdma_cntl = RREG32(reg_offset);
1382                 sdma_cntl = REG_SET_FIELD(sdma_cntl, SDMA0_CNTL, TRAP_ENABLE,
1383                                state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
1384                 WREG32(reg_offset, sdma_cntl);
1385         }
1386
1387         return 0;
1388 }
1389
1390 static int sdma_v5_2_process_trap_irq(struct amdgpu_device *adev,
1391                                       struct amdgpu_irq_src *source,
1392                                       struct amdgpu_iv_entry *entry)
1393 {
1394         uint32_t mes_queue_id = entry->src_data[0];
1395
1396         DRM_DEBUG("IH: SDMA trap\n");
1397
1398         if (adev->enable_mes && (mes_queue_id & AMDGPU_FENCE_MES_QUEUE_FLAG)) {
1399                 struct amdgpu_mes_queue *queue;
1400
1401                 mes_queue_id &= AMDGPU_FENCE_MES_QUEUE_ID_MASK;
1402
1403                 spin_lock(&adev->mes.queue_id_lock);
1404                 queue = idr_find(&adev->mes.queue_id_idr, mes_queue_id);
1405                 if (queue) {
1406                         DRM_DEBUG("process smda queue id = %d\n", mes_queue_id);
1407                         amdgpu_fence_process(queue->ring);
1408                 }
1409                 spin_unlock(&adev->mes.queue_id_lock);
1410                 return 0;
1411         }
1412
1413         switch (entry->client_id) {
1414         case SOC15_IH_CLIENTID_SDMA0:
1415                 switch (entry->ring_id) {
1416                 case 0:
1417                         amdgpu_fence_process(&adev->sdma.instance[0].ring);
1418                         break;
1419                 case 1:
1420                         /* XXX compute */
1421                         break;
1422                 case 2:
1423                         /* XXX compute */
1424                         break;
1425                 case 3:
1426                         /* XXX page queue*/
1427                         break;
1428                 }
1429                 break;
1430         case SOC15_IH_CLIENTID_SDMA1:
1431                 switch (entry->ring_id) {
1432                 case 0:
1433                         amdgpu_fence_process(&adev->sdma.instance[1].ring);
1434                         break;
1435                 case 1:
1436                         /* XXX compute */
1437                         break;
1438                 case 2:
1439                         /* XXX compute */
1440                         break;
1441                 case 3:
1442                         /* XXX page queue*/
1443                         break;
1444                 }
1445                 break;
1446         case SOC15_IH_CLIENTID_SDMA2:
1447                 switch (entry->ring_id) {
1448                 case 0:
1449                         amdgpu_fence_process(&adev->sdma.instance[2].ring);
1450                         break;
1451                 case 1:
1452                         /* XXX compute */
1453                         break;
1454                 case 2:
1455                         /* XXX compute */
1456                         break;
1457                 case 3:
1458                         /* XXX page queue*/
1459                         break;
1460                 }
1461                 break;
1462         case SOC15_IH_CLIENTID_SDMA3_Sienna_Cichlid:
1463                 switch (entry->ring_id) {
1464                 case 0:
1465                         amdgpu_fence_process(&adev->sdma.instance[3].ring);
1466                         break;
1467                 case 1:
1468                         /* XXX compute */
1469                         break;
1470                 case 2:
1471                         /* XXX compute */
1472                         break;
1473                 case 3:
1474                         /* XXX page queue*/
1475                         break;
1476                 }
1477                 break;
1478         }
1479         return 0;
1480 }
1481
1482 static int sdma_v5_2_process_illegal_inst_irq(struct amdgpu_device *adev,
1483                                               struct amdgpu_irq_src *source,
1484                                               struct amdgpu_iv_entry *entry)
1485 {
1486         return 0;
1487 }
1488
1489 static bool sdma_v5_2_firmware_mgcg_support(struct amdgpu_device *adev,
1490                                                      int i)
1491 {
1492         switch (amdgpu_ip_version(adev, SDMA0_HWIP, 0)) {
1493         case IP_VERSION(5, 2, 1):
1494                 if (adev->sdma.instance[i].fw_version < 70)
1495                         return false;
1496                 break;
1497         case IP_VERSION(5, 2, 3):
1498                 if (adev->sdma.instance[i].fw_version < 47)
1499                         return false;
1500                 break;
1501         case IP_VERSION(5, 2, 7):
1502                 if (adev->sdma.instance[i].fw_version < 9)
1503                         return false;
1504                 break;
1505         default:
1506                 return true;
1507         }
1508
1509         return true;
1510
1511 }
1512
1513 static void sdma_v5_2_update_medium_grain_clock_gating(struct amdgpu_device *adev,
1514                                                        bool enable)
1515 {
1516         uint32_t data, def;
1517         int i;
1518
1519         for (i = 0; i < adev->sdma.num_instances; i++) {
1520
1521                 if (!sdma_v5_2_firmware_mgcg_support(adev, i))
1522                         adev->cg_flags &= ~AMD_CG_SUPPORT_SDMA_MGCG;
1523
1524                 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_SDMA_MGCG)) {
1525                         /* Enable sdma clock gating */
1526                         def = data = RREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_CLK_CTRL));
1527                         data &= ~(SDMA0_CLK_CTRL__SOFT_OVERRIDE4_MASK |
1528                                   SDMA0_CLK_CTRL__SOFT_OVERRIDE3_MASK |
1529                                   SDMA0_CLK_CTRL__SOFT_OVERRIDE2_MASK |
1530                                   SDMA0_CLK_CTRL__SOFT_OVERRIDE1_MASK |
1531                                   SDMA0_CLK_CTRL__SOFT_OVERRIDE0_MASK |
1532                                   SDMA0_CLK_CTRL__SOFT_OVERRIDER_REG_MASK);
1533                         if (def != data)
1534                                 WREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_CLK_CTRL), data);
1535                 } else {
1536                         /* Disable sdma clock gating */
1537                         def = data = RREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_CLK_CTRL));
1538                         data |= (SDMA0_CLK_CTRL__SOFT_OVERRIDE4_MASK |
1539                                  SDMA0_CLK_CTRL__SOFT_OVERRIDE3_MASK |
1540                                  SDMA0_CLK_CTRL__SOFT_OVERRIDE2_MASK |
1541                                  SDMA0_CLK_CTRL__SOFT_OVERRIDE1_MASK |
1542                                  SDMA0_CLK_CTRL__SOFT_OVERRIDE0_MASK |
1543                                  SDMA0_CLK_CTRL__SOFT_OVERRIDER_REG_MASK);
1544                         if (def != data)
1545                                 WREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_CLK_CTRL), data);
1546                 }
1547         }
1548 }
1549
1550 static void sdma_v5_2_update_medium_grain_light_sleep(struct amdgpu_device *adev,
1551                                                       bool enable)
1552 {
1553         uint32_t data, def;
1554         int i;
1555
1556         for (i = 0; i < adev->sdma.num_instances; i++) {
1557                 if (adev->sdma.instance[i].fw_version < 70 &&
1558                     amdgpu_ip_version(adev, SDMA0_HWIP, 0) ==
1559                             IP_VERSION(5, 2, 1))
1560                         adev->cg_flags &= ~AMD_CG_SUPPORT_SDMA_LS;
1561
1562                 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_SDMA_LS)) {
1563                         /* Enable sdma mem light sleep */
1564                         def = data = RREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_POWER_CNTL));
1565                         data |= SDMA0_POWER_CNTL__MEM_POWER_OVERRIDE_MASK;
1566                         if (def != data)
1567                                 WREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_POWER_CNTL), data);
1568
1569                 } else {
1570                         /* Disable sdma mem light sleep */
1571                         def = data = RREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_POWER_CNTL));
1572                         data &= ~SDMA0_POWER_CNTL__MEM_POWER_OVERRIDE_MASK;
1573                         if (def != data)
1574                                 WREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_POWER_CNTL), data);
1575
1576                 }
1577         }
1578 }
1579
1580 static int sdma_v5_2_set_clockgating_state(void *handle,
1581                                            enum amd_clockgating_state state)
1582 {
1583         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1584
1585         if (amdgpu_sriov_vf(adev))
1586                 return 0;
1587
1588         switch (amdgpu_ip_version(adev, SDMA0_HWIP, 0)) {
1589         case IP_VERSION(5, 2, 0):
1590         case IP_VERSION(5, 2, 2):
1591         case IP_VERSION(5, 2, 1):
1592         case IP_VERSION(5, 2, 4):
1593         case IP_VERSION(5, 2, 5):
1594         case IP_VERSION(5, 2, 6):
1595         case IP_VERSION(5, 2, 3):
1596         case IP_VERSION(5, 2, 7):
1597                 sdma_v5_2_update_medium_grain_clock_gating(adev,
1598                                 state == AMD_CG_STATE_GATE);
1599                 sdma_v5_2_update_medium_grain_light_sleep(adev,
1600                                 state == AMD_CG_STATE_GATE);
1601                 break;
1602         default:
1603                 break;
1604         }
1605
1606         return 0;
1607 }
1608
1609 static int sdma_v5_2_set_powergating_state(void *handle,
1610                                           enum amd_powergating_state state)
1611 {
1612         return 0;
1613 }
1614
1615 static void sdma_v5_2_get_clockgating_state(void *handle, u64 *flags)
1616 {
1617         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1618         int data;
1619
1620         if (amdgpu_sriov_vf(adev))
1621                 *flags = 0;
1622
1623         /* AMD_CG_SUPPORT_SDMA_MGCG */
1624         data = RREG32(sdma_v5_2_get_reg_offset(adev, 0, mmSDMA0_CLK_CTRL));
1625         if (!(data & SDMA0_CLK_CTRL__CGCG_EN_OVERRIDE_MASK))
1626                 *flags |= AMD_CG_SUPPORT_SDMA_MGCG;
1627
1628         /* AMD_CG_SUPPORT_SDMA_LS */
1629         data = RREG32_KIQ(sdma_v5_2_get_reg_offset(adev, 0, mmSDMA0_POWER_CNTL));
1630         if (data & SDMA0_POWER_CNTL__MEM_POWER_OVERRIDE_MASK)
1631                 *flags |= AMD_CG_SUPPORT_SDMA_LS;
1632 }
1633
1634 static void sdma_v5_2_ring_begin_use(struct amdgpu_ring *ring)
1635 {
1636         struct amdgpu_device *adev = ring->adev;
1637
1638         /* SDMA 5.2.3 (RMB) FW doesn't seem to properly
1639          * disallow GFXOFF in some cases leading to
1640          * hangs in SDMA.  Disallow GFXOFF while SDMA is active.
1641          * We can probably just limit this to 5.2.3,
1642          * but it shouldn't hurt for other parts since
1643          * this GFXOFF will be disallowed anyway when SDMA is
1644          * active, this just makes it explicit.
1645          */
1646         amdgpu_gfx_off_ctrl(adev, false);
1647 }
1648
1649 static void sdma_v5_2_ring_end_use(struct amdgpu_ring *ring)
1650 {
1651         struct amdgpu_device *adev = ring->adev;
1652
1653         /* SDMA 5.2.3 (RMB) FW doesn't seem to properly
1654          * disallow GFXOFF in some cases leading to
1655          * hangs in SDMA.  Allow GFXOFF when SDMA is complete.
1656          */
1657         amdgpu_gfx_off_ctrl(adev, true);
1658 }
1659
1660 const struct amd_ip_funcs sdma_v5_2_ip_funcs = {
1661         .name = "sdma_v5_2",
1662         .early_init = sdma_v5_2_early_init,
1663         .late_init = NULL,
1664         .sw_init = sdma_v5_2_sw_init,
1665         .sw_fini = sdma_v5_2_sw_fini,
1666         .hw_init = sdma_v5_2_hw_init,
1667         .hw_fini = sdma_v5_2_hw_fini,
1668         .suspend = sdma_v5_2_suspend,
1669         .resume = sdma_v5_2_resume,
1670         .is_idle = sdma_v5_2_is_idle,
1671         .wait_for_idle = sdma_v5_2_wait_for_idle,
1672         .soft_reset = sdma_v5_2_soft_reset,
1673         .set_clockgating_state = sdma_v5_2_set_clockgating_state,
1674         .set_powergating_state = sdma_v5_2_set_powergating_state,
1675         .get_clockgating_state = sdma_v5_2_get_clockgating_state,
1676 };
1677
1678 static const struct amdgpu_ring_funcs sdma_v5_2_ring_funcs = {
1679         .type = AMDGPU_RING_TYPE_SDMA,
1680         .align_mask = 0xf,
1681         .nop = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP),
1682         .support_64bit_ptrs = true,
1683         .secure_submission_supported = true,
1684         .get_rptr = sdma_v5_2_ring_get_rptr,
1685         .get_wptr = sdma_v5_2_ring_get_wptr,
1686         .set_wptr = sdma_v5_2_ring_set_wptr,
1687         .emit_frame_size =
1688                 5 + /* sdma_v5_2_ring_init_cond_exec */
1689                 6 + /* sdma_v5_2_ring_emit_hdp_flush */
1690                 3 + /* hdp_invalidate */
1691                 6 + /* sdma_v5_2_ring_emit_pipeline_sync */
1692                 /* sdma_v5_2_ring_emit_vm_flush */
1693                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +
1694                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 6 +
1695                 10 + 10 + 10, /* sdma_v5_2_ring_emit_fence x3 for user fence, vm fence */
1696         .emit_ib_size = 7 + 6, /* sdma_v5_2_ring_emit_ib */
1697         .emit_ib = sdma_v5_2_ring_emit_ib,
1698         .emit_mem_sync = sdma_v5_2_ring_emit_mem_sync,
1699         .emit_fence = sdma_v5_2_ring_emit_fence,
1700         .emit_pipeline_sync = sdma_v5_2_ring_emit_pipeline_sync,
1701         .emit_vm_flush = sdma_v5_2_ring_emit_vm_flush,
1702         .emit_hdp_flush = sdma_v5_2_ring_emit_hdp_flush,
1703         .test_ring = sdma_v5_2_ring_test_ring,
1704         .test_ib = sdma_v5_2_ring_test_ib,
1705         .insert_nop = sdma_v5_2_ring_insert_nop,
1706         .pad_ib = sdma_v5_2_ring_pad_ib,
1707         .begin_use = sdma_v5_2_ring_begin_use,
1708         .end_use = sdma_v5_2_ring_end_use,
1709         .emit_wreg = sdma_v5_2_ring_emit_wreg,
1710         .emit_reg_wait = sdma_v5_2_ring_emit_reg_wait,
1711         .emit_reg_write_reg_wait = sdma_v5_2_ring_emit_reg_write_reg_wait,
1712         .init_cond_exec = sdma_v5_2_ring_init_cond_exec,
1713         .preempt_ib = sdma_v5_2_ring_preempt_ib,
1714 };
1715
1716 static void sdma_v5_2_set_ring_funcs(struct amdgpu_device *adev)
1717 {
1718         int i;
1719
1720         for (i = 0; i < adev->sdma.num_instances; i++) {
1721                 adev->sdma.instance[i].ring.funcs = &sdma_v5_2_ring_funcs;
1722                 adev->sdma.instance[i].ring.me = i;
1723         }
1724 }
1725
1726 static const struct amdgpu_irq_src_funcs sdma_v5_2_trap_irq_funcs = {
1727         .set = sdma_v5_2_set_trap_irq_state,
1728         .process = sdma_v5_2_process_trap_irq,
1729 };
1730
1731 static const struct amdgpu_irq_src_funcs sdma_v5_2_illegal_inst_irq_funcs = {
1732         .process = sdma_v5_2_process_illegal_inst_irq,
1733 };
1734
1735 static void sdma_v5_2_set_irq_funcs(struct amdgpu_device *adev)
1736 {
1737         adev->sdma.trap_irq.num_types = AMDGPU_SDMA_IRQ_INSTANCE0 +
1738                                         adev->sdma.num_instances;
1739         adev->sdma.trap_irq.funcs = &sdma_v5_2_trap_irq_funcs;
1740         adev->sdma.illegal_inst_irq.funcs = &sdma_v5_2_illegal_inst_irq_funcs;
1741 }
1742
1743 /**
1744  * sdma_v5_2_emit_copy_buffer - copy buffer using the sDMA engine
1745  *
1746  * @ib: indirect buffer to copy to
1747  * @src_offset: src GPU address
1748  * @dst_offset: dst GPU address
1749  * @byte_count: number of bytes to xfer
1750  * @tmz: if a secure copy should be used
1751  *
1752  * Copy GPU buffers using the DMA engine.
1753  * Used by the amdgpu ttm implementation to move pages if
1754  * registered as the asic copy callback.
1755  */
1756 static void sdma_v5_2_emit_copy_buffer(struct amdgpu_ib *ib,
1757                                        uint64_t src_offset,
1758                                        uint64_t dst_offset,
1759                                        uint32_t byte_count,
1760                                        bool tmz)
1761 {
1762         ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_COPY) |
1763                 SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR) |
1764                 SDMA_PKT_COPY_LINEAR_HEADER_TMZ(tmz ? 1 : 0);
1765         ib->ptr[ib->length_dw++] = byte_count - 1;
1766         ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */
1767         ib->ptr[ib->length_dw++] = lower_32_bits(src_offset);
1768         ib->ptr[ib->length_dw++] = upper_32_bits(src_offset);
1769         ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset);
1770         ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset);
1771 }
1772
1773 /**
1774  * sdma_v5_2_emit_fill_buffer - fill buffer using the sDMA engine
1775  *
1776  * @ib: indirect buffer to fill
1777  * @src_data: value to write to buffer
1778  * @dst_offset: dst GPU address
1779  * @byte_count: number of bytes to xfer
1780  *
1781  * Fill GPU buffers using the DMA engine.
1782  */
1783 static void sdma_v5_2_emit_fill_buffer(struct amdgpu_ib *ib,
1784                                        uint32_t src_data,
1785                                        uint64_t dst_offset,
1786                                        uint32_t byte_count)
1787 {
1788         ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_CONST_FILL);
1789         ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset);
1790         ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset);
1791         ib->ptr[ib->length_dw++] = src_data;
1792         ib->ptr[ib->length_dw++] = byte_count - 1;
1793 }
1794
1795 static const struct amdgpu_buffer_funcs sdma_v5_2_buffer_funcs = {
1796         .copy_max_bytes = 0x400000,
1797         .copy_num_dw = 7,
1798         .emit_copy_buffer = sdma_v5_2_emit_copy_buffer,
1799
1800         .fill_max_bytes = 0x400000,
1801         .fill_num_dw = 5,
1802         .emit_fill_buffer = sdma_v5_2_emit_fill_buffer,
1803 };
1804
1805 static void sdma_v5_2_set_buffer_funcs(struct amdgpu_device *adev)
1806 {
1807         if (adev->mman.buffer_funcs == NULL) {
1808                 adev->mman.buffer_funcs = &sdma_v5_2_buffer_funcs;
1809                 adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring;
1810         }
1811 }
1812
1813 static const struct amdgpu_vm_pte_funcs sdma_v5_2_vm_pte_funcs = {
1814         .copy_pte_num_dw = 7,
1815         .copy_pte = sdma_v5_2_vm_copy_pte,
1816         .write_pte = sdma_v5_2_vm_write_pte,
1817         .set_pte_pde = sdma_v5_2_vm_set_pte_pde,
1818 };
1819
1820 static void sdma_v5_2_set_vm_pte_funcs(struct amdgpu_device *adev)
1821 {
1822         unsigned i;
1823
1824         if (adev->vm_manager.vm_pte_funcs == NULL) {
1825                 adev->vm_manager.vm_pte_funcs = &sdma_v5_2_vm_pte_funcs;
1826                 for (i = 0; i < adev->sdma.num_instances; i++) {
1827                         adev->vm_manager.vm_pte_scheds[i] =
1828                                 &adev->sdma.instance[i].ring.sched;
1829                 }
1830                 adev->vm_manager.vm_pte_num_scheds = adev->sdma.num_instances;
1831         }
1832 }
1833
1834 const struct amdgpu_ip_block_version sdma_v5_2_ip_block = {
1835         .type = AMD_IP_BLOCK_TYPE_SDMA,
1836         .major = 5,
1837         .minor = 2,
1838         .rev = 0,
1839         .funcs = &sdma_v5_2_ip_funcs,
1840 };