drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c

   1 /*
   2  * Copyright 2020 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23
  24 #include <linux/delay.h>
  25 #include <linux/firmware.h>
  26 #include <linux/module.h>
  27 #include <linux/pci.h>
  28
  29 #include "amdgpu.h"
  30 #include "amdgpu_ucode.h"
  31 #include "amdgpu_trace.h"
  32
  33 #include "gc/gc_11_0_0_offset.h"
  34 #include "gc/gc_11_0_0_sh_mask.h"
  35 #include "gc/gc_11_0_0_default.h"
  36 #include "hdp/hdp_6_0_0_offset.h"
  37 #include "ivsrcid/gfx/irqsrcs_gfx_11_0_0.h"
  38
  39 #include "soc15_common.h"
  40 #include "soc15.h"
  41 #include "sdma_v6_0_0_pkt_open.h"
  42 #include "nbio_v4_3.h"
  43 #include "sdma_common.h"
  44 #include "sdma_v6_0.h"
  45 #include "v11_structs.h"
  46
  47 MODULE_FIRMWARE("amdgpu/sdma_6_0_0.bin");
  48 MODULE_FIRMWARE("amdgpu/sdma_6_0_1.bin");
  49 MODULE_FIRMWARE("amdgpu/sdma_6_0_2.bin");
  50 MODULE_FIRMWARE("amdgpu/sdma_6_0_3.bin");
  51 MODULE_FIRMWARE("amdgpu/sdma_6_1_0.bin");
  52 MODULE_FIRMWARE("amdgpu/sdma_6_1_1.bin");
  53 MODULE_FIRMWARE("amdgpu/sdma_6_1_2.bin");
  54
  55 #define SDMA1_REG_OFFSET 0x600
  56 #define SDMA0_HYP_DEC_REG_START 0x5880
  57 #define SDMA0_HYP_DEC_REG_END 0x589a
  58 #define SDMA1_HYP_DEC_REG_OFFSET 0x20
  59
  60 static void sdma_v6_0_set_ring_funcs(struct amdgpu_device *adev);
  61 static void sdma_v6_0_set_buffer_funcs(struct amdgpu_device *adev);
  62 static void sdma_v6_0_set_vm_pte_funcs(struct amdgpu_device *adev);
  63 static void sdma_v6_0_set_irq_funcs(struct amdgpu_device *adev);
  64 static int sdma_v6_0_start(struct amdgpu_device *adev);
  65
  66 static u32 sdma_v6_0_get_reg_offset(struct amdgpu_device *adev, u32 instance, u32 internal_offset)
  67 {
  68         u32 base;
  69
  70         if (internal_offset >= SDMA0_HYP_DEC_REG_START &&
  71             internal_offset <= SDMA0_HYP_DEC_REG_END) {
  72                 base = adev->reg_offset[GC_HWIP][0][1];
  73                 if (instance != 0)
  74                         internal_offset += SDMA1_HYP_DEC_REG_OFFSET * instance;
  75         } else {
  76                 base = adev->reg_offset[GC_HWIP][0][0];
  77                 if (instance == 1)
  78                         internal_offset += SDMA1_REG_OFFSET;
  79         }
  80
  81         return base + internal_offset;
  82 }
  83
  84 static unsigned sdma_v6_0_ring_init_cond_exec(struct amdgpu_ring *ring,
  85                                               uint64_t addr)
  86 {
  87         unsigned ret;
  88
  89         amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_COND_EXE));
  90         amdgpu_ring_write(ring, lower_32_bits(addr));
  91         amdgpu_ring_write(ring, upper_32_bits(addr));
  92         amdgpu_ring_write(ring, 1);
  93         /* this is the offset we need patch later */
  94         ret = ring->wptr & ring->buf_mask;
  95         /* insert dummy here and patch it later */
  96         amdgpu_ring_write(ring, 0);
  97
  98         return ret;
  99 }
 100
 101 /**
 102  * sdma_v6_0_ring_get_rptr - get the current read pointer
 103  *
 104  * @ring: amdgpu ring pointer
 105  *
 106  * Get the current rptr from the hardware.
 107  */
 108 static uint64_t sdma_v6_0_ring_get_rptr(struct amdgpu_ring *ring)
 109 {
 110         u64 *rptr;
 111
 112         /* XXX check if swapping is necessary on BE */
 113         rptr = (u64 *)ring->rptr_cpu_addr;
 114
 115         DRM_DEBUG("rptr before shift == 0x%016llx\n", *rptr);
 116         return ((*rptr) >> 2);
 117 }
 118
 119 /**
 120  * sdma_v6_0_ring_get_wptr - get the current write pointer
 121  *
 122  * @ring: amdgpu ring pointer
 123  *
 124  * Get the current wptr from the hardware.
 125  */
 126 static uint64_t sdma_v6_0_ring_get_wptr(struct amdgpu_ring *ring)
 127 {
 128         u64 wptr = 0;
 129
 130         if (ring->use_doorbell) {
 131                 /* XXX check if swapping is necessary on BE */
 132                 wptr = READ_ONCE(*((u64 *)ring->wptr_cpu_addr));
 133                 DRM_DEBUG("wptr/doorbell before shift == 0x%016llx\n", wptr);
 134         }
 135
 136         return wptr >> 2;
 137 }
 138
 139 /**
 140  * sdma_v6_0_ring_set_wptr - commit the write pointer
 141  *
 142  * @ring: amdgpu ring pointer
 143  *
 144  * Write the wptr back to the hardware.
 145  */
 146 static void sdma_v6_0_ring_set_wptr(struct amdgpu_ring *ring)
 147 {
 148         struct amdgpu_device *adev = ring->adev;
 149
 150         if (ring->use_doorbell) {
 151                 DRM_DEBUG("Using doorbell -- "
 152                           "wptr_offs == 0x%08x "
 153                           "lower_32_bits(ring->wptr) << 2 == 0x%08x "
 154                           "upper_32_bits(ring->wptr) << 2 == 0x%08x\n",
 155                           ring->wptr_offs,
 156                           lower_32_bits(ring->wptr << 2),
 157                           upper_32_bits(ring->wptr << 2));
 158                 /* XXX check if swapping is necessary on BE */
 159                 atomic64_set((atomic64_t *)ring->wptr_cpu_addr,
 160                              ring->wptr << 2);
 161                 DRM_DEBUG("calling WDOORBELL64(0x%08x, 0x%016llx)\n",
 162                           ring->doorbell_index, ring->wptr << 2);
 163                 WDOORBELL64(ring->doorbell_index, ring->wptr << 2);
 164         } else {
 165                 DRM_DEBUG("Not using doorbell -- "
 166                           "regSDMA%i_GFX_RB_WPTR == 0x%08x "
 167                           "regSDMA%i_GFX_RB_WPTR_HI == 0x%08x\n",
 168                           ring->me,
 169                           lower_32_bits(ring->wptr << 2),
 170                           ring->me,
 171                           upper_32_bits(ring->wptr << 2));
 172                 WREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev,
 173                                                              ring->me, regSDMA0_QUEUE0_RB_WPTR),
 174                                 lower_32_bits(ring->wptr << 2));
 175                 WREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev,
 176                                                              ring->me, regSDMA0_QUEUE0_RB_WPTR_HI),
 177                                 upper_32_bits(ring->wptr << 2));
 178         }
 179 }
 180
 181 static void sdma_v6_0_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)
 182 {
 183         struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring);
 184         int i;
 185
 186         for (i = 0; i < count; i++)
 187                 if (sdma && sdma->burst_nop && (i == 0))
 188                         amdgpu_ring_write(ring, ring->funcs->nop |
 189                                 SDMA_PKT_NOP_HEADER_COUNT(count - 1));
 190                 else
 191                         amdgpu_ring_write(ring, ring->funcs->nop);
 192 }
 193
 194 /*
 195  * sdma_v6_0_ring_emit_ib - Schedule an IB on the DMA engine
 196  *
 197  * @ring: amdgpu ring pointer
 198  * @ib: IB object to schedule
 199  * @flags: unused
 200  * @job: job to retrieve vmid from
 201  *
 202  * Schedule an IB in the DMA ring.
 203  */
 204 static void sdma_v6_0_ring_emit_ib(struct amdgpu_ring *ring,
 205                                    struct amdgpu_job *job,
 206                                    struct amdgpu_ib *ib,
 207                                    uint32_t flags)
 208 {
 209         unsigned vmid = AMDGPU_JOB_GET_VMID(job);
 210         uint64_t csa_mc_addr = amdgpu_sdma_get_csa_mc_addr(ring, vmid);
 211
 212         /* An IB packet must end on a 8 DW boundary--the next dword
 213          * must be on a 8-dword boundary. Our IB packet below is 6
 214          * dwords long, thus add x number of NOPs, such that, in
 215          * modular arithmetic,
 216          * wptr + 6 + x = 8k, k >= 0, which in C is,
 217          * (wptr + 6 + x) % 8 = 0.
 218          * The expression below, is a solution of x.
 219          */
 220         sdma_v6_0_ring_insert_nop(ring, (2 - lower_32_bits(ring->wptr)) & 7);
 221
 222         amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_INDIRECT) |
 223                           SDMA_PKT_INDIRECT_HEADER_VMID(vmid & 0xf));
 224         /* base must be 32 byte aligned */
 225         amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr) & 0xffffffe0);
 226         amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
 227         amdgpu_ring_write(ring, ib->length_dw);
 228         amdgpu_ring_write(ring, lower_32_bits(csa_mc_addr));
 229         amdgpu_ring_write(ring, upper_32_bits(csa_mc_addr));
 230 }
 231
 232 /**
 233  * sdma_v6_0_ring_emit_mem_sync - flush the IB by graphics cache rinse
 234  *
 235  * @ring: amdgpu ring pointer
 236  *
 237  * flush the IB by graphics cache rinse.
 238  */
 239 static void sdma_v6_0_ring_emit_mem_sync(struct amdgpu_ring *ring)
 240 {
 241         uint32_t gcr_cntl = SDMA_GCR_GL2_INV | SDMA_GCR_GL2_WB | SDMA_GCR_GLM_INV |
 242                             SDMA_GCR_GL1_INV | SDMA_GCR_GLV_INV | SDMA_GCR_GLK_INV |
 243                             SDMA_GCR_GLI_INV(1);
 244
 245         /* flush entire cache L0/L1/L2, this can be optimized by performance requirement */
 246         amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_GCR_REQ));
 247         amdgpu_ring_write(ring, SDMA_PKT_GCR_REQ_PAYLOAD1_BASE_VA_31_7(0));
 248         amdgpu_ring_write(ring, SDMA_PKT_GCR_REQ_PAYLOAD2_GCR_CONTROL_15_0(gcr_cntl) |
 249                           SDMA_PKT_GCR_REQ_PAYLOAD2_BASE_VA_47_32(0));
 250         amdgpu_ring_write(ring, SDMA_PKT_GCR_REQ_PAYLOAD3_LIMIT_VA_31_7(0) |
 251                           SDMA_PKT_GCR_REQ_PAYLOAD3_GCR_CONTROL_18_16(gcr_cntl >> 16));
 252         amdgpu_ring_write(ring, SDMA_PKT_GCR_REQ_PAYLOAD4_LIMIT_VA_47_32(0) |
 253                           SDMA_PKT_GCR_REQ_PAYLOAD4_VMID(0));
 254 }
 255
 256
 257 /**
 258  * sdma_v6_0_ring_emit_hdp_flush - emit an hdp flush on the DMA ring
 259  *
 260  * @ring: amdgpu ring pointer
 261  *
 262  * Emit an hdp flush packet on the requested DMA ring.
 263  */
 264 static void sdma_v6_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
 265 {
 266         struct amdgpu_device *adev = ring->adev;
 267         u32 ref_and_mask = 0;
 268         const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg;
 269
 270         ref_and_mask = nbio_hf_reg->ref_and_mask_sdma0 << ring->me;
 271
 272         amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_POLL_REGMEM) |
 273                           SDMA_PKT_POLL_REGMEM_HEADER_HDP_FLUSH(1) |
 274                           SDMA_PKT_POLL_REGMEM_HEADER_FUNC(3)); /* == */
 275         amdgpu_ring_write(ring, (adev->nbio.funcs->get_hdp_flush_done_offset(adev)) << 2);
 276         amdgpu_ring_write(ring, (adev->nbio.funcs->get_hdp_flush_req_offset(adev)) << 2);
 277         amdgpu_ring_write(ring, ref_and_mask); /* reference */
 278         amdgpu_ring_write(ring, ref_and_mask); /* mask */
 279         amdgpu_ring_write(ring, SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff) |
 280                           SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(10)); /* retry count, poll interval */
 281 }
 282
 283 /**
 284  * sdma_v6_0_ring_emit_fence - emit a fence on the DMA ring
 285  *
 286  * @ring: amdgpu ring pointer
 287  * @addr: address
 288  * @seq: fence seq number
 289  * @flags: fence flags
 290  *
 291  * Add a DMA fence packet to the ring to write
 292  * the fence seq number and DMA trap packet to generate
 293  * an interrupt if needed.
 294  */
 295 static void sdma_v6_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq,
 296                                       unsigned flags)
 297 {
 298         bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
 299         /* write the fence */
 300         amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_FENCE) |
 301                           SDMA_PKT_FENCE_HEADER_MTYPE(0x3)); /* Ucached(UC) */
 302         /* zero in first two bits */
 303         BUG_ON(addr & 0x3);
 304         amdgpu_ring_write(ring, lower_32_bits(addr));
 305         amdgpu_ring_write(ring, upper_32_bits(addr));
 306         amdgpu_ring_write(ring, lower_32_bits(seq));
 307
 308         /* optionally write high bits as well */
 309         if (write64bit) {
 310                 addr += 4;
 311                 amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_FENCE) |
 312                                   SDMA_PKT_FENCE_HEADER_MTYPE(0x3));
 313                 /* zero in first two bits */
 314                 BUG_ON(addr & 0x3);
 315                 amdgpu_ring_write(ring, lower_32_bits(addr));
 316                 amdgpu_ring_write(ring, upper_32_bits(addr));
 317                 amdgpu_ring_write(ring, upper_32_bits(seq));
 318         }
 319
 320         if (flags & AMDGPU_FENCE_FLAG_INT) {
 321                 uint32_t ctx = ring->is_mes_queue ?
 322                         (ring->hw_queue_id | AMDGPU_FENCE_MES_QUEUE_FLAG) : 0;
 323                 /* generate an interrupt */
 324                 amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_TRAP));
 325                 amdgpu_ring_write(ring, SDMA_PKT_TRAP_INT_CONTEXT_INT_CONTEXT(ctx));
 326         }
 327 }
 328
 329 /**
 330  * sdma_v6_0_gfx_stop - stop the gfx async dma engines
 331  *
 332  * @adev: amdgpu_device pointer
 333  *
 334  * Stop the gfx async dma ring buffers.
 335  */
 336 static void sdma_v6_0_gfx_stop(struct amdgpu_device *adev)
 337 {
 338         u32 rb_cntl, ib_cntl;
 339         int i;
 340
 341         for (i = 0; i < adev->sdma.num_instances; i++) {
 342                 rb_cntl = RREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_CNTL));
 343                 rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_QUEUE0_RB_CNTL, RB_ENABLE, 0);
 344                 WREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_CNTL), rb_cntl);
 345                 ib_cntl = RREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_IB_CNTL));
 346                 ib_cntl = REG_SET_FIELD(ib_cntl, SDMA0_QUEUE0_IB_CNTL, IB_ENABLE, 0);
 347                 WREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_IB_CNTL), ib_cntl);
 348         }
 349 }
 350
 351 /**
 352  * sdma_v6_0_rlc_stop - stop the compute async dma engines
 353  *
 354  * @adev: amdgpu_device pointer
 355  *
 356  * Stop the compute async dma queues.
 357  */
 358 static void sdma_v6_0_rlc_stop(struct amdgpu_device *adev)
 359 {
 360         /* XXX todo */
 361 }
 362
 363 /**
 364  * sdma_v6_0_ctxempty_int_enable - enable or disable context empty interrupts
 365  *
 366  * @adev: amdgpu_device pointer
 367  * @enable: enable/disable context switching due to queue empty conditions
 368  *
 369  * Enable or disable the async dma engines queue empty context switch.
 370  */
 371 static void sdma_v6_0_ctxempty_int_enable(struct amdgpu_device *adev, bool enable)
 372 {
 373         u32 f32_cntl;
 374         int i;
 375
 376         if (!amdgpu_sriov_vf(adev)) {
 377                 for (i = 0; i < adev->sdma.num_instances; i++) {
 378                         f32_cntl = RREG32(sdma_v6_0_get_reg_offset(adev, i, regSDMA0_CNTL));
 379                         f32_cntl = REG_SET_FIELD(f32_cntl, SDMA0_CNTL,
 380                                         CTXEMPTY_INT_ENABLE, enable ? 1 : 0);
 381                         WREG32(sdma_v6_0_get_reg_offset(adev, i, regSDMA0_CNTL), f32_cntl);
 382                 }
 383         }
 384 }
 385
 386 /**
 387  * sdma_v6_0_enable - stop the async dma engines
 388  *
 389  * @adev: amdgpu_device pointer
 390  * @enable: enable/disable the DMA MEs.
 391  *
 392  * Halt or unhalt the async dma engines.
 393  */
 394 static void sdma_v6_0_enable(struct amdgpu_device *adev, bool enable)
 395 {
 396         u32 f32_cntl;
 397         int i;
 398
 399         if (!enable) {
 400                 sdma_v6_0_gfx_stop(adev);
 401                 sdma_v6_0_rlc_stop(adev);
 402         }
 403
 404         if (amdgpu_sriov_vf(adev))
 405                 return;
 406
 407         for (i = 0; i < adev->sdma.num_instances; i++) {
 408                 f32_cntl = RREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_F32_CNTL));
 409                 f32_cntl = REG_SET_FIELD(f32_cntl, SDMA0_F32_CNTL, HALT, enable ? 0 : 1);
 410                 WREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_F32_CNTL), f32_cntl);
 411         }
 412 }
 413
 414 /**
 415  * sdma_v6_0_gfx_resume - setup and start the async dma engines
 416  *
 417  * @adev: amdgpu_device pointer
 418  *
 419  * Set up the gfx DMA ring buffers and enable them.
 420  * Returns 0 for success, error for failure.
 421  */
 422 static int sdma_v6_0_gfx_resume(struct amdgpu_device *adev)
 423 {
 424         struct amdgpu_ring *ring;
 425         u32 rb_cntl, ib_cntl;
 426         u32 rb_bufsz;
 427         u32 doorbell;
 428         u32 doorbell_offset;
 429         u32 temp;
 430         u64 wptr_gpu_addr;
 431         int i, r;
 432
 433         for (i = 0; i < adev->sdma.num_instances; i++) {
 434                 ring = &adev->sdma.instance[i].ring;
 435
 436                 if (!amdgpu_sriov_vf(adev))
 437                         WREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_SEM_WAIT_FAIL_TIMER_CNTL), 0);
 438
 439                 /* Set ring buffer size in dwords */
 440                 rb_bufsz = order_base_2(ring->ring_size / 4);
 441                 rb_cntl = RREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_CNTL));
 442                 rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_QUEUE0_RB_CNTL, RB_SIZE, rb_bufsz);
 443 #ifdef __BIG_ENDIAN
 444                 rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_QUEUE0_RB_CNTL, RB_SWAP_ENABLE, 1);
 445                 rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_QUEUE0_RB_CNTL,
 446                                         RPTR_WRITEBACK_SWAP_ENABLE, 1);
 447 #endif
 448                 rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_QUEUE0_RB_CNTL, RB_PRIV, 1);
 449                 WREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_CNTL), rb_cntl);
 450
 451                 /* Initialize the ring buffer's read and write pointers */
 452                 WREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_RPTR), 0);
 453                 WREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_RPTR_HI), 0);
 454                 WREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_WPTR), 0);
 455                 WREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_WPTR_HI), 0);
 456
 457                 /* setup the wptr shadow polling */
 458                 wptr_gpu_addr = ring->wptr_gpu_addr;
 459                 WREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_WPTR_POLL_ADDR_LO),
 460                        lower_32_bits(wptr_gpu_addr));
 461                 WREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_WPTR_POLL_ADDR_HI),
 462                        upper_32_bits(wptr_gpu_addr));
 463
 464                 /* set the wb address whether it's enabled or not */
 465                 WREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_RPTR_ADDR_HI),
 466                        upper_32_bits(ring->rptr_gpu_addr) & 0xFFFFFFFF);
 467                 WREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_RPTR_ADDR_LO),
 468                        lower_32_bits(ring->rptr_gpu_addr) & 0xFFFFFFFC);
 469
 470                 rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_QUEUE0_RB_CNTL, RPTR_WRITEBACK_ENABLE, 1);
 471                 rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_QUEUE0_RB_CNTL, WPTR_POLL_ENABLE, 0);
 472                 rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_QUEUE0_RB_CNTL, F32_WPTR_POLL_ENABLE, 1);
 473
 474                 WREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_BASE), ring->gpu_addr >> 8);
 475                 WREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_BASE_HI), ring->gpu_addr >> 40);
 476
 477                 ring->wptr = 0;
 478
 479                 /* before programing wptr to a less value, need set minor_ptr_update first */
 480                 WREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_MINOR_PTR_UPDATE), 1);
 481
 482                 if (!amdgpu_sriov_vf(adev)) { /* only bare-metal use register write for wptr */
 483                         WREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_WPTR), lower_32_bits(ring->wptr) << 2);
 484                         WREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_WPTR_HI), upper_32_bits(ring->wptr) << 2);
 485                 }
 486
 487                 doorbell = RREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_DOORBELL));
 488                 doorbell_offset = RREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_DOORBELL_OFFSET));
 489
 490                 if (ring->use_doorbell) {
 491                         doorbell = REG_SET_FIELD(doorbell, SDMA0_QUEUE0_DOORBELL, ENABLE, 1);
 492                         doorbell_offset = REG_SET_FIELD(doorbell_offset, SDMA0_QUEUE0_DOORBELL_OFFSET,
 493                                         OFFSET, ring->doorbell_index);
 494                 } else {
 495                         doorbell = REG_SET_FIELD(doorbell, SDMA0_QUEUE0_DOORBELL, ENABLE, 0);
 496                 }
 497                 WREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_DOORBELL), doorbell);
 498                 WREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_DOORBELL_OFFSET), doorbell_offset);
 499
 500                 if (i == 0)
 501                         adev->nbio.funcs->sdma_doorbell_range(adev, i, ring->use_doorbell,
 502                                                       ring->doorbell_index,
 503                                                       adev->doorbell_index.sdma_doorbell_range * adev->sdma.num_instances);
 504
 505                 if (amdgpu_sriov_vf(adev))
 506                         sdma_v6_0_ring_set_wptr(ring);
 507
 508                 /* set minor_ptr_update to 0 after wptr programed */
 509                 WREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_MINOR_PTR_UPDATE), 0);
 510
 511                 /* Set up sdma hang watchdog */
 512                 temp = RREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_WATCHDOG_CNTL));
 513                 /* 100ms per unit */
 514                 temp = REG_SET_FIELD(temp, SDMA0_WATCHDOG_CNTL, QUEUE_HANG_COUNT,
 515                                      max(adev->usec_timeout/100000, 1));
 516                 WREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_WATCHDOG_CNTL), temp);
 517
 518                 /* Set up RESP_MODE to non-copy addresses */
 519                 temp = RREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_UTCL1_CNTL));
 520                 temp = REG_SET_FIELD(temp, SDMA0_UTCL1_CNTL, RESP_MODE, 3);
 521                 temp = REG_SET_FIELD(temp, SDMA0_UTCL1_CNTL, REDO_DELAY, 9);
 522                 WREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_UTCL1_CNTL), temp);
 523
 524                 /* program default cache read and write policy */
 525                 temp = RREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_UTCL1_PAGE));
 526                 /* clean read policy and write policy bits */
 527                 temp &= 0xFF0FFF;
 528                 temp |= ((CACHE_READ_POLICY_L2__DEFAULT << 12) |
 529                          (CACHE_WRITE_POLICY_L2__DEFAULT << 14) |
 530                          SDMA0_UTCL1_PAGE__LLC_NOALLOC_MASK);
 531                 WREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_UTCL1_PAGE), temp);
 532
 533                 if (!amdgpu_sriov_vf(adev)) {
 534                         /* unhalt engine */
 535                         temp = RREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_F32_CNTL));
 536                         temp = REG_SET_FIELD(temp, SDMA0_F32_CNTL, HALT, 0);
 537                         temp = REG_SET_FIELD(temp, SDMA0_F32_CNTL, TH1_RESET, 0);
 538                         WREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_F32_CNTL), temp);
 539                 }
 540
 541                 /* enable DMA RB */
 542                 rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_QUEUE0_RB_CNTL, RB_ENABLE, 1);
 543                 WREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_CNTL), rb_cntl);
 544
 545                 ib_cntl = RREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_IB_CNTL));
 546                 ib_cntl = REG_SET_FIELD(ib_cntl, SDMA0_QUEUE0_IB_CNTL, IB_ENABLE, 1);
 547 #ifdef __BIG_ENDIAN
 548                 ib_cntl = REG_SET_FIELD(ib_cntl, SDMA0_QUEUE0_IB_CNTL, IB_SWAP_ENABLE, 1);
 549 #endif
 550                 /* enable DMA IBs */
 551                 WREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_IB_CNTL), ib_cntl);
 552
 553                 if (amdgpu_sriov_vf(adev))
 554                         sdma_v6_0_enable(adev, true);
 555
 556                 r = amdgpu_ring_test_helper(ring);
 557                 if (r)
 558                         return r;
 559         }
 560
 561         return 0;
 562 }
 563
 564 /**
 565  * sdma_v6_0_rlc_resume - setup and start the async dma engines
 566  *
 567  * @adev: amdgpu_device pointer
 568  *
 569  * Set up the compute DMA queues and enable them.
 570  * Returns 0 for success, error for failure.
 571  */
 572 static int sdma_v6_0_rlc_resume(struct amdgpu_device *adev)
 573 {
 574         return 0;
 575 }
 576
 577 /**
 578  * sdma_v6_0_load_microcode - load the sDMA ME ucode
 579  *
 580  * @adev: amdgpu_device pointer
 581  *
 582  * Loads the sDMA0/1 ucode.
 583  * Returns 0 for success, -EINVAL if the ucode is not available.
 584  */
 585 static int sdma_v6_0_load_microcode(struct amdgpu_device *adev)
 586 {
 587         const struct sdma_firmware_header_v2_0 *hdr;
 588         const __le32 *fw_data;
 589         u32 fw_size;
 590         int i, j;
 591         bool use_broadcast;
 592
 593         /* halt the MEs */
 594         sdma_v6_0_enable(adev, false);
 595
 596         if (!adev->sdma.instance[0].fw)
 597                 return -EINVAL;
 598
 599         /* use broadcast mode to load SDMA microcode by default */
 600         use_broadcast = true;
 601
 602         if (use_broadcast) {
 603                 dev_info(adev->dev, "Use broadcast method to load SDMA firmware\n");
 604                 /* load Control Thread microcode */
 605                 hdr = (const struct sdma_firmware_header_v2_0 *)adev->sdma.instance[0].fw->data;
 606                 amdgpu_ucode_print_sdma_hdr(&hdr->header);
 607                 fw_size = le32_to_cpu(hdr->ctx_jt_offset + hdr->ctx_jt_size) / 4;
 608
 609                 fw_data = (const __le32 *)
 610                         (adev->sdma.instance[0].fw->data +
 611                                 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
 612
 613                 WREG32(sdma_v6_0_get_reg_offset(adev, 0, regSDMA0_BROADCAST_UCODE_ADDR), 0);
 614
 615                 for (j = 0; j < fw_size; j++) {
 616                         if (amdgpu_emu_mode == 1 && j % 500 == 0)
 617                                 msleep(1);
 618                         WREG32(sdma_v6_0_get_reg_offset(adev, 0, regSDMA0_BROADCAST_UCODE_DATA), le32_to_cpup(fw_data++));
 619                 }
 620
 621                 /* load Context Switch microcode */
 622                 fw_size = le32_to_cpu(hdr->ctl_jt_offset + hdr->ctl_jt_size) / 4;
 623
 624                 fw_data = (const __le32 *)
 625                         (adev->sdma.instance[0].fw->data +
 626                                 le32_to_cpu(hdr->ctl_ucode_offset));
 627
 628                 WREG32(sdma_v6_0_get_reg_offset(adev, 0, regSDMA0_BROADCAST_UCODE_ADDR), 0x8000);
 629
 630                 for (j = 0; j < fw_size; j++) {
 631                         if (amdgpu_emu_mode == 1 && j % 500 == 0)
 632                                 msleep(1);
 633                         WREG32(sdma_v6_0_get_reg_offset(adev, 0, regSDMA0_BROADCAST_UCODE_DATA), le32_to_cpup(fw_data++));
 634                 }
 635         } else {
 636                 dev_info(adev->dev, "Use legacy method to load SDMA firmware\n");
 637                 for (i = 0; i < adev->sdma.num_instances; i++) {
 638                         /* load Control Thread microcode */
 639                         hdr = (const struct sdma_firmware_header_v2_0 *)adev->sdma.instance[0].fw->data;
 640                         amdgpu_ucode_print_sdma_hdr(&hdr->header);
 641                         fw_size = le32_to_cpu(hdr->ctx_jt_offset + hdr->ctx_jt_size) / 4;
 642
 643                         fw_data = (const __le32 *)
 644                                 (adev->sdma.instance[0].fw->data +
 645                                         le32_to_cpu(hdr->header.ucode_array_offset_bytes));
 646
 647                         WREG32(sdma_v6_0_get_reg_offset(adev, i, regSDMA0_UCODE_ADDR), 0);
 648
 649                         for (j = 0; j < fw_size; j++) {
 650                                 if (amdgpu_emu_mode == 1 && j % 500 == 0)
 651                                         msleep(1);
 652                                 WREG32(sdma_v6_0_get_reg_offset(adev, i, regSDMA0_UCODE_DATA), le32_to_cpup(fw_data++));
 653                         }
 654
 655                         WREG32(sdma_v6_0_get_reg_offset(adev, i, regSDMA0_UCODE_ADDR), adev->sdma.instance[0].fw_version);
 656
 657                         /* load Context Switch microcode */
 658                         fw_size = le32_to_cpu(hdr->ctl_jt_offset + hdr->ctl_jt_size) / 4;
 659
 660                         fw_data = (const __le32 *)
 661                                 (adev->sdma.instance[0].fw->data +
 662                                         le32_to_cpu(hdr->ctl_ucode_offset));
 663
 664                         WREG32(sdma_v6_0_get_reg_offset(adev, i, regSDMA0_UCODE_ADDR), 0x8000);
 665
 666                         for (j = 0; j < fw_size; j++) {
 667                                 if (amdgpu_emu_mode == 1 && j % 500 == 0)
 668                                         msleep(1);
 669                                 WREG32(sdma_v6_0_get_reg_offset(adev, i, regSDMA0_UCODE_DATA), le32_to_cpup(fw_data++));
 670                         }
 671
 672                         WREG32(sdma_v6_0_get_reg_offset(adev, i, regSDMA0_UCODE_ADDR), adev->sdma.instance[0].fw_version);
 673                 }
 674         }
 675
 676         return 0;
 677 }
 678
 679 static int sdma_v6_0_soft_reset(void *handle)
 680 {
 681         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 682         u32 tmp;
 683         int i;
 684
 685         sdma_v6_0_gfx_stop(adev);
 686
 687         for (i = 0; i < adev->sdma.num_instances; i++) {
 688                 tmp = RREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_FREEZE));
 689                 tmp |= SDMA0_FREEZE__FREEZE_MASK;
 690                 WREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_FREEZE), tmp);
 691                 tmp = RREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_F32_CNTL));
 692                 tmp |= SDMA0_F32_CNTL__HALT_MASK;
 693                 tmp |= SDMA0_F32_CNTL__TH1_RESET_MASK;
 694                 WREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_F32_CNTL), tmp);
 695
 696                 WREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_PREEMPT), 0);
 697
 698                 udelay(100);
 699
 700                 tmp = GRBM_SOFT_RESET__SOFT_RESET_SDMA0_MASK << i;
 701                 WREG32_SOC15(GC, 0, regGRBM_SOFT_RESET, tmp);
 702                 tmp = RREG32_SOC15(GC, 0, regGRBM_SOFT_RESET);
 703
 704                 udelay(100);
 705
 706                 WREG32_SOC15(GC, 0, regGRBM_SOFT_RESET, 0);
 707                 tmp = RREG32_SOC15(GC, 0, regGRBM_SOFT_RESET);
 708
 709                 udelay(100);
 710         }
 711
 712         return sdma_v6_0_start(adev);
 713 }
 714
 715 static bool sdma_v6_0_check_soft_reset(void *handle)
 716 {
 717         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 718         struct amdgpu_ring *ring;
 719         int i, r;
 720         long tmo = msecs_to_jiffies(1000);
 721
 722         for (i = 0; i < adev->sdma.num_instances; i++) {
 723                 ring = &adev->sdma.instance[i].ring;
 724                 r = amdgpu_ring_test_ib(ring, tmo);
 725                 if (r)
 726                         return true;
 727         }
 728
 729         return false;
 730 }
 731
 732 /**
 733  * sdma_v6_0_start - setup and start the async dma engines
 734  *
 735  * @adev: amdgpu_device pointer
 736  *
 737  * Set up the DMA engines and enable them.
 738  * Returns 0 for success, error for failure.
 739  */
 740 static int sdma_v6_0_start(struct amdgpu_device *adev)
 741 {
 742         int r = 0;
 743
 744         if (amdgpu_sriov_vf(adev)) {
 745                 sdma_v6_0_enable(adev, false);
 746
 747                 /* set RB registers */
 748                 r = sdma_v6_0_gfx_resume(adev);
 749                 return r;
 750         }
 751
 752         if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
 753                 r = sdma_v6_0_load_microcode(adev);
 754                 if (r)
 755                         return r;
 756
 757                 /* The value of regSDMA_F32_CNTL is invalid the moment after loading fw */
 758                 if (amdgpu_emu_mode == 1)
 759                         msleep(1000);
 760         }
 761
 762         /* unhalt the MEs */
 763         sdma_v6_0_enable(adev, true);
 764         /* enable sdma ring preemption */
 765         sdma_v6_0_ctxempty_int_enable(adev, true);
 766
 767         /* start the gfx rings and rlc compute queues */
 768         r = sdma_v6_0_gfx_resume(adev);
 769         if (r)
 770                 return r;
 771         r = sdma_v6_0_rlc_resume(adev);
 772
 773         return r;
 774 }
 775
 776 static int sdma_v6_0_mqd_init(struct amdgpu_device *adev, void *mqd,
 777                               struct amdgpu_mqd_prop *prop)
 778 {
 779         struct v11_sdma_mqd *m = mqd;
 780         uint64_t wb_gpu_addr;
 781
 782         m->sdmax_rlcx_rb_cntl =
 783                 order_base_2(prop->queue_size / 4) << SDMA0_QUEUE0_RB_CNTL__RB_SIZE__SHIFT |
 784                 1 << SDMA0_QUEUE0_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT |
 785                 4 << SDMA0_QUEUE0_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT |
 786                 1 << SDMA0_QUEUE0_RB_CNTL__F32_WPTR_POLL_ENABLE__SHIFT;
 787
 788         m->sdmax_rlcx_rb_base = lower_32_bits(prop->hqd_base_gpu_addr >> 8);
 789         m->sdmax_rlcx_rb_base_hi = upper_32_bits(prop->hqd_base_gpu_addr >> 8);
 790
 791         wb_gpu_addr = prop->wptr_gpu_addr;
 792         m->sdmax_rlcx_rb_wptr_poll_addr_lo = lower_32_bits(wb_gpu_addr);
 793         m->sdmax_rlcx_rb_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr);
 794
 795         wb_gpu_addr = prop->rptr_gpu_addr;
 796         m->sdmax_rlcx_rb_rptr_addr_lo = lower_32_bits(wb_gpu_addr);
 797         m->sdmax_rlcx_rb_rptr_addr_hi = upper_32_bits(wb_gpu_addr);
 798
 799         m->sdmax_rlcx_ib_cntl = RREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, 0,
 800                                                         regSDMA0_QUEUE0_IB_CNTL));
 801
 802         m->sdmax_rlcx_doorbell_offset =
 803                 prop->doorbell_index << SDMA0_QUEUE0_DOORBELL_OFFSET__OFFSET__SHIFT;
 804
 805         m->sdmax_rlcx_doorbell = REG_SET_FIELD(0, SDMA0_QUEUE0_DOORBELL, ENABLE, 1);
 806
 807         m->sdmax_rlcx_skip_cntl = 0;
 808         m->sdmax_rlcx_context_status = 0;
 809         m->sdmax_rlcx_doorbell_log = 0;
 810
 811         m->sdmax_rlcx_rb_aql_cntl = regSDMA0_QUEUE0_RB_AQL_CNTL_DEFAULT;
 812         m->sdmax_rlcx_dummy_reg = regSDMA0_QUEUE0_DUMMY_REG_DEFAULT;
 813
 814         return 0;
 815 }
 816
 817 static void sdma_v6_0_set_mqd_funcs(struct amdgpu_device *adev)
 818 {
 819         adev->mqds[AMDGPU_HW_IP_DMA].mqd_size = sizeof(struct v11_sdma_mqd);
 820         adev->mqds[AMDGPU_HW_IP_DMA].init_mqd = sdma_v6_0_mqd_init;
 821 }
 822
 823 /**
 824  * sdma_v6_0_ring_test_ring - simple async dma engine test
 825  *
 826  * @ring: amdgpu_ring structure holding ring information
 827  *
 828  * Test the DMA engine by writing using it to write an
 829  * value to memory.
 830  * Returns 0 for success, error for failure.
 831  */
 832 static int sdma_v6_0_ring_test_ring(struct amdgpu_ring *ring)
 833 {
 834         struct amdgpu_device *adev = ring->adev;
 835         unsigned i;
 836         unsigned index;
 837         int r;
 838         u32 tmp;
 839         u64 gpu_addr;
 840         volatile uint32_t *cpu_ptr = NULL;
 841
 842         tmp = 0xCAFEDEAD;
 843
 844         if (ring->is_mes_queue) {
 845                 uint32_t offset = 0;
 846                 offset = amdgpu_mes_ctx_get_offs(ring,
 847                                          AMDGPU_MES_CTX_PADDING_OFFS);
 848                 gpu_addr = amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset);
 849                 cpu_ptr = amdgpu_mes_ctx_get_offs_cpu_addr(ring, offset);
 850                 *cpu_ptr = tmp;
 851         } else {
 852                 r = amdgpu_device_wb_get(adev, &index);
 853                 if (r) {
 854                         dev_err(adev->dev, "(%d) failed to allocate wb slot\n", r);
 855                         return r;
 856                 }
 857
 858                 gpu_addr = adev->wb.gpu_addr + (index * 4);
 859                 adev->wb.wb[index] = cpu_to_le32(tmp);
 860         }
 861
 862         r = amdgpu_ring_alloc(ring, 5);
 863         if (r) {
 864                 DRM_ERROR("amdgpu: dma failed to lock ring %d (%d).\n", ring->idx, r);
 865                 if (!ring->is_mes_queue)
 866                         amdgpu_device_wb_free(adev, index);
 867                 return r;
 868         }
 869
 870         amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_WRITE) |
 871                           SDMA_PKT_COPY_LINEAR_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR));
 872         amdgpu_ring_write(ring, lower_32_bits(gpu_addr));
 873         amdgpu_ring_write(ring, upper_32_bits(gpu_addr));
 874         amdgpu_ring_write(ring, SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(0));
 875         amdgpu_ring_write(ring, 0xDEADBEEF);
 876         amdgpu_ring_commit(ring);
 877
 878         for (i = 0; i < adev->usec_timeout; i++) {
 879                 if (ring->is_mes_queue)
 880                         tmp = le32_to_cpu(*cpu_ptr);
 881                 else
 882                         tmp = le32_to_cpu(adev->wb.wb[index]);
 883                 if (tmp == 0xDEADBEEF)
 884                         break;
 885                 if (amdgpu_emu_mode == 1)
 886                         msleep(1);
 887                 else
 888                         udelay(1);
 889         }
 890
 891         if (i >= adev->usec_timeout)
 892                 r = -ETIMEDOUT;
 893
 894         if (!ring->is_mes_queue)
 895                 amdgpu_device_wb_free(adev, index);
 896
 897         return r;
 898 }
 899
 900 /*
 901  * sdma_v6_0_ring_test_ib - test an IB on the DMA engine
 902  *
 903  * @ring: amdgpu_ring structure holding ring information
 904  * @timeout: timeout value in jiffies, or MAX_SCHEDULE_TIMEOUT
 905  *
 906  * Test a simple IB in the DMA ring.
 907  * Returns 0 on success, error on failure.
 908  */
 909 static int sdma_v6_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 910 {
 911         struct amdgpu_device *adev = ring->adev;
 912         struct amdgpu_ib ib;
 913         struct dma_fence *f = NULL;
 914         unsigned index;
 915         long r;
 916         u32 tmp = 0;
 917         u64 gpu_addr;
 918         volatile uint32_t *cpu_ptr = NULL;
 919
 920         tmp = 0xCAFEDEAD;
 921         memset(&ib, 0, sizeof(ib));
 922
 923         if (ring->is_mes_queue) {
 924                 uint32_t offset = 0;
 925                 offset = amdgpu_mes_ctx_get_offs(ring, AMDGPU_MES_CTX_IB_OFFS);
 926                 ib.gpu_addr = amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset);
 927                 ib.ptr = (void *)amdgpu_mes_ctx_get_offs_cpu_addr(ring, offset);
 928
 929                 offset = amdgpu_mes_ctx_get_offs(ring,
 930                                          AMDGPU_MES_CTX_PADDING_OFFS);
 931                 gpu_addr = amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset);
 932                 cpu_ptr = amdgpu_mes_ctx_get_offs_cpu_addr(ring, offset);
 933                 *cpu_ptr = tmp;
 934         } else {
 935                 r = amdgpu_device_wb_get(adev, &index);
 936                 if (r) {
 937                         dev_err(adev->dev, "(%ld) failed to allocate wb slot\n", r);
 938                         return r;
 939                 }
 940
 941                 gpu_addr = adev->wb.gpu_addr + (index * 4);
 942                 adev->wb.wb[index] = cpu_to_le32(tmp);
 943
 944                 r = amdgpu_ib_get(adev, NULL, 256, AMDGPU_IB_POOL_DIRECT, &ib);
 945                 if (r) {
 946                         DRM_ERROR("amdgpu: failed to get ib (%ld).\n", r);
 947                         goto err0;
 948                 }
 949         }
 950
 951         ib.ptr[0] = SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_WRITE) |
 952                 SDMA_PKT_COPY_LINEAR_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR);
 953         ib.ptr[1] = lower_32_bits(gpu_addr);
 954         ib.ptr[2] = upper_32_bits(gpu_addr);
 955         ib.ptr[3] = SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(0);
 956         ib.ptr[4] = 0xDEADBEEF;
 957         ib.ptr[5] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
 958         ib.ptr[6] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
 959         ib.ptr[7] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
 960         ib.length_dw = 8;
 961
 962         r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
 963         if (r)
 964                 goto err1;
 965
 966         r = dma_fence_wait_timeout(f, false, timeout);
 967         if (r == 0) {
 968                 DRM_ERROR("amdgpu: IB test timed out\n");
 969                 r = -ETIMEDOUT;
 970                 goto err1;
 971         } else if (r < 0) {
 972                 DRM_ERROR("amdgpu: fence wait failed (%ld).\n", r);
 973                 goto err1;
 974         }
 975
 976         if (ring->is_mes_queue)
 977                 tmp = le32_to_cpu(*cpu_ptr);
 978         else
 979                 tmp = le32_to_cpu(adev->wb.wb[index]);
 980
 981         if (tmp == 0xDEADBEEF)
 982                 r = 0;
 983         else
 984                 r = -EINVAL;
 985
 986 err1:
 987         amdgpu_ib_free(adev, &ib, NULL);
 988         dma_fence_put(f);
 989 err0:
 990         if (!ring->is_mes_queue)
 991                 amdgpu_device_wb_free(adev, index);
 992         return r;
 993 }
 994
 995
 996 /**
 997  * sdma_v6_0_vm_copy_pte - update PTEs by copying them from the GART
 998  *
 999  * @ib: indirect buffer to fill with commands
1000  * @pe: addr of the page entry
1001  * @src: src addr to copy from
1002  * @count: number of page entries to update
1003  *
1004  * Update PTEs by copying them from the GART using sDMA.
1005  */
1006 static void sdma_v6_0_vm_copy_pte(struct amdgpu_ib *ib,
1007                                   uint64_t pe, uint64_t src,
1008                                   unsigned count)
1009 {
1010         unsigned bytes = count * 8;
1011
1012         ib->ptr[ib->length_dw++] = SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_COPY) |
1013                 SDMA_PKT_COPY_LINEAR_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR);
1014         ib->ptr[ib->length_dw++] = bytes - 1;
1015         ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */
1016         ib->ptr[ib->length_dw++] = lower_32_bits(src);
1017         ib->ptr[ib->length_dw++] = upper_32_bits(src);
1018         ib->ptr[ib->length_dw++] = lower_32_bits(pe);
1019         ib->ptr[ib->length_dw++] = upper_32_bits(pe);
1020
1021 }
1022
1023 /**
1024  * sdma_v6_0_vm_write_pte - update PTEs by writing them manually
1025  *
1026  * @ib: indirect buffer to fill with commands
1027  * @pe: addr of the page entry
1028  * @value: dst addr to write into pe
1029  * @count: number of page entries to update
1030  * @incr: increase next addr by incr bytes
1031  *
1032  * Update PTEs by writing them manually using sDMA.
1033  */
1034 static void sdma_v6_0_vm_write_pte(struct amdgpu_ib *ib, uint64_t pe,
1035                                    uint64_t value, unsigned count,
1036                                    uint32_t incr)
1037 {
1038         unsigned ndw = count * 2;
1039
1040         ib->ptr[ib->length_dw++] = SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_WRITE) |
1041                 SDMA_PKT_COPY_LINEAR_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR);
1042         ib->ptr[ib->length_dw++] = lower_32_bits(pe);
1043         ib->ptr[ib->length_dw++] = upper_32_bits(pe);
1044         ib->ptr[ib->length_dw++] = ndw - 1;
1045         for (; ndw > 0; ndw -= 2) {
1046                 ib->ptr[ib->length_dw++] = lower_32_bits(value);
1047                 ib->ptr[ib->length_dw++] = upper_32_bits(value);
1048                 value += incr;
1049         }
1050 }
1051
1052 /**
1053  * sdma_v6_0_vm_set_pte_pde - update the page tables using sDMA
1054  *
1055  * @ib: indirect buffer to fill with commands
1056  * @pe: addr of the page entry
1057  * @addr: dst addr to write into pe
1058  * @count: number of page entries to update
1059  * @incr: increase next addr by incr bytes
1060  * @flags: access flags
1061  *
1062  * Update the page tables using sDMA.
1063  */
1064 static void sdma_v6_0_vm_set_pte_pde(struct amdgpu_ib *ib,
1065                                      uint64_t pe,
1066                                      uint64_t addr, unsigned count,
1067                                      uint32_t incr, uint64_t flags)
1068 {
1069         /* for physically contiguous pages (vram) */
1070         ib->ptr[ib->length_dw++] = SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_PTEPDE);
1071         ib->ptr[ib->length_dw++] = lower_32_bits(pe); /* dst addr */
1072         ib->ptr[ib->length_dw++] = upper_32_bits(pe);
1073         ib->ptr[ib->length_dw++] = lower_32_bits(flags); /* mask */
1074         ib->ptr[ib->length_dw++] = upper_32_bits(flags);
1075         ib->ptr[ib->length_dw++] = lower_32_bits(addr); /* value */
1076         ib->ptr[ib->length_dw++] = upper_32_bits(addr);
1077         ib->ptr[ib->length_dw++] = incr; /* increment size */
1078         ib->ptr[ib->length_dw++] = 0;
1079         ib->ptr[ib->length_dw++] = count - 1; /* number of entries */
1080 }
1081
1082 /*
1083  * sdma_v6_0_ring_pad_ib - pad the IB
1084  * @ib: indirect buffer to fill with padding
1085  * @ring: amdgpu ring pointer
1086  *
1087  * Pad the IB with NOPs to a boundary multiple of 8.
1088  */
1089 static void sdma_v6_0_ring_pad_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib)
1090 {
1091         struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring);
1092         u32 pad_count;
1093         int i;
1094
1095         pad_count = (-ib->length_dw) & 0x7;
1096         for (i = 0; i < pad_count; i++)
1097                 if (sdma && sdma->burst_nop && (i == 0))
1098                         ib->ptr[ib->length_dw++] =
1099                                 SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_NOP) |
1100                                 SDMA_PKT_NOP_HEADER_COUNT(pad_count - 1);
1101                 else
1102                         ib->ptr[ib->length_dw++] =
1103                                 SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_NOP);
1104 }
1105
1106 /**
1107  * sdma_v6_0_ring_emit_pipeline_sync - sync the pipeline
1108  *
1109  * @ring: amdgpu_ring pointer
1110  *
1111  * Make sure all previous operations are completed (CIK).
1112  */
1113 static void sdma_v6_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
1114 {
1115         uint32_t seq = ring->fence_drv.sync_seq;
1116         uint64_t addr = ring->fence_drv.gpu_addr;
1117
1118         /* wait for idle */
1119         amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_POLL_REGMEM) |
1120                           SDMA_PKT_POLL_REGMEM_HEADER_HDP_FLUSH(0) |
1121                           SDMA_PKT_POLL_REGMEM_HEADER_FUNC(3) | /* equal */
1122                           SDMA_PKT_POLL_REGMEM_HEADER_MEM_POLL(1));
1123         amdgpu_ring_write(ring, addr & 0xfffffffc);
1124         amdgpu_ring_write(ring, upper_32_bits(addr) & 0xffffffff);
1125         amdgpu_ring_write(ring, seq); /* reference */
1126         amdgpu_ring_write(ring, 0xffffffff); /* mask */
1127         amdgpu_ring_write(ring, SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff) |
1128                           SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(4)); /* retry count, poll interval */
1129 }
1130
1131 /*
1132  * sdma_v6_0_ring_emit_vm_flush - vm flush using sDMA
1133  *
1134  * @ring: amdgpu_ring pointer
1135  * @vmid: vmid number to use
1136  * @pd_addr: address
1137  *
1138  * Update the page table base and flush the VM TLB
1139  * using sDMA.
1140  */
1141 static void sdma_v6_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
1142                                          unsigned vmid, uint64_t pd_addr)
1143 {
1144         struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->vm_hub];
1145         uint32_t req = hub->vmhub_funcs->get_invalidate_req(vmid, 0);
1146
1147         /* Update the PD address for this VMID. */
1148         amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_lo32 +
1149                               (hub->ctx_addr_distance * vmid),
1150                               lower_32_bits(pd_addr));
1151         amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_hi32 +
1152                               (hub->ctx_addr_distance * vmid),
1153                               upper_32_bits(pd_addr));
1154
1155         /* Trigger invalidation. */
1156         amdgpu_ring_write(ring,
1157                           SDMA_PKT_VM_INVALIDATION_HEADER_OP(SDMA_OP_POLL_REGMEM) |
1158                           SDMA_PKT_VM_INVALIDATION_HEADER_SUB_OP(SDMA_SUBOP_VM_INVALIDATION) |
1159                           SDMA_PKT_VM_INVALIDATION_HEADER_GFX_ENG_ID(ring->vm_inv_eng) |
1160                           SDMA_PKT_VM_INVALIDATION_HEADER_MM_ENG_ID(0x1f));
1161         amdgpu_ring_write(ring, req);
1162         amdgpu_ring_write(ring, 0xFFFFFFFF);
1163         amdgpu_ring_write(ring,
1164                           SDMA_PKT_VM_INVALIDATION_ADDRESSRANGEHI_INVALIDATEACK(1 << vmid) |
1165                           SDMA_PKT_VM_INVALIDATION_ADDRESSRANGEHI_ADDRESSRANGEHI(0x1F));
1166 }
1167
1168 static void sdma_v6_0_ring_emit_wreg(struct amdgpu_ring *ring,
1169                                      uint32_t reg, uint32_t val)
1170 {
1171         amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_SRBM_WRITE) |
1172                           SDMA_PKT_SRBM_WRITE_HEADER_BYTE_EN(0xf));
1173         amdgpu_ring_write(ring, reg);
1174         amdgpu_ring_write(ring, val);
1175 }
1176
1177 static void sdma_v6_0_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
1178                                          uint32_t val, uint32_t mask)
1179 {
1180         amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_POLL_REGMEM) |
1181                           SDMA_PKT_POLL_REGMEM_HEADER_HDP_FLUSH(0) |
1182                           SDMA_PKT_POLL_REGMEM_HEADER_FUNC(3)); /* equal */
1183         amdgpu_ring_write(ring, reg << 2);
1184         amdgpu_ring_write(ring, 0);
1185         amdgpu_ring_write(ring, val); /* reference */
1186         amdgpu_ring_write(ring, mask); /* mask */
1187         amdgpu_ring_write(ring, SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff) |
1188                           SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(10));
1189 }
1190
1191 static void sdma_v6_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring,
1192                                                    uint32_t reg0, uint32_t reg1,
1193                                                    uint32_t ref, uint32_t mask)
1194 {
1195         amdgpu_ring_emit_wreg(ring, reg0, ref);
1196         /* wait for a cycle to reset vm_inv_eng*_ack */
1197         amdgpu_ring_emit_reg_wait(ring, reg0, 0, 0);
1198         amdgpu_ring_emit_reg_wait(ring, reg1, mask, mask);
1199 }
1200
1201 static struct amdgpu_sdma_ras sdma_v6_0_3_ras = {
1202         .ras_block = {
1203                 .ras_late_init = amdgpu_ras_block_late_init,
1204         },
1205 };
1206
1207 static void sdma_v6_0_set_ras_funcs(struct amdgpu_device *adev)
1208 {
1209         switch (amdgpu_ip_version(adev, SDMA0_HWIP, 0)) {
1210         case IP_VERSION(6, 0, 3):
1211                 adev->sdma.ras = &sdma_v6_0_3_ras;
1212                 break;
1213         default:
1214                 break;
1215         }
1216 }
1217
1218 static int sdma_v6_0_early_init(void *handle)
1219 {
1220         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1221         int r;
1222
1223         r = amdgpu_sdma_init_microcode(adev, 0, true);
1224         if (r)
1225                 return r;
1226
1227         sdma_v6_0_set_ring_funcs(adev);
1228         sdma_v6_0_set_buffer_funcs(adev);
1229         sdma_v6_0_set_vm_pte_funcs(adev);
1230         sdma_v6_0_set_irq_funcs(adev);
1231         sdma_v6_0_set_mqd_funcs(adev);
1232         sdma_v6_0_set_ras_funcs(adev);
1233
1234         return 0;
1235 }
1236
1237 static int sdma_v6_0_sw_init(void *handle)
1238 {
1239         struct amdgpu_ring *ring;
1240         int r, i;
1241         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1242
1243         /* SDMA trap event */
1244         r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GFX,
1245                               GFX_11_0_0__SRCID__SDMA_TRAP,
1246                               &adev->sdma.trap_irq);
1247         if (r)
1248                 return r;
1249
1250         for (i = 0; i < adev->sdma.num_instances; i++) {
1251                 ring = &adev->sdma.instance[i].ring;
1252                 ring->ring_obj = NULL;
1253                 ring->use_doorbell = true;
1254                 ring->me = i;
1255
1256                 DRM_DEBUG("SDMA %d use_doorbell being set to: [%s]\n", i,
1257                                 ring->use_doorbell?"true":"false");
1258
1259                 ring->doorbell_index =
1260                         (adev->doorbell_index.sdma_engine[i] << 1); // get DWORD offset
1261
1262                 ring->vm_hub = AMDGPU_GFXHUB(0);
1263                 sprintf(ring->name, "sdma%d", i);
1264                 r = amdgpu_ring_init(adev, ring, 1024,
1265                                      &adev->sdma.trap_irq,
1266                                      AMDGPU_SDMA_IRQ_INSTANCE0 + i,
1267                                      AMDGPU_RING_PRIO_DEFAULT, NULL);
1268                 if (r)
1269                         return r;
1270         }
1271
1272         if (amdgpu_sdma_ras_sw_init(adev)) {
1273                 dev_err(adev->dev, "Failed to initialize sdma ras block!\n");
1274                 return -EINVAL;
1275         }
1276
1277         return r;
1278 }
1279
1280 static int sdma_v6_0_sw_fini(void *handle)
1281 {
1282         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1283         int i;
1284
1285         for (i = 0; i < adev->sdma.num_instances; i++)
1286                 amdgpu_ring_fini(&adev->sdma.instance[i].ring);
1287
1288         amdgpu_sdma_destroy_inst_ctx(adev, true);
1289
1290         return 0;
1291 }
1292
1293 static int sdma_v6_0_hw_init(void *handle)
1294 {
1295         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1296
1297         return sdma_v6_0_start(adev);
1298 }
1299
1300 static int sdma_v6_0_hw_fini(void *handle)
1301 {
1302         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1303
1304         if (amdgpu_sriov_vf(adev))
1305                 return 0;
1306
1307         sdma_v6_0_ctxempty_int_enable(adev, false);
1308         sdma_v6_0_enable(adev, false);
1309
1310         return 0;
1311 }
1312
1313 static int sdma_v6_0_suspend(void *handle)
1314 {
1315         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1316
1317         return sdma_v6_0_hw_fini(adev);
1318 }
1319
1320 static int sdma_v6_0_resume(void *handle)
1321 {
1322         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1323
1324         return sdma_v6_0_hw_init(adev);
1325 }
1326
1327 static bool sdma_v6_0_is_idle(void *handle)
1328 {
1329         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1330         u32 i;
1331
1332         for (i = 0; i < adev->sdma.num_instances; i++) {
1333                 u32 tmp = RREG32(sdma_v6_0_get_reg_offset(adev, i, regSDMA0_STATUS_REG));
1334
1335                 if (!(tmp & SDMA0_STATUS_REG__IDLE_MASK))
1336                         return false;
1337         }
1338
1339         return true;
1340 }
1341
1342 static int sdma_v6_0_wait_for_idle(void *handle)
1343 {
1344         unsigned i;
1345         u32 sdma0, sdma1;
1346         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1347
1348         for (i = 0; i < adev->usec_timeout; i++) {
1349                 sdma0 = RREG32(sdma_v6_0_get_reg_offset(adev, 0, regSDMA0_STATUS_REG));
1350                 sdma1 = RREG32(sdma_v6_0_get_reg_offset(adev, 1, regSDMA0_STATUS_REG));
1351
1352                 if (sdma0 & sdma1 & SDMA0_STATUS_REG__IDLE_MASK)
1353                         return 0;
1354                 udelay(1);
1355         }
1356         return -ETIMEDOUT;
1357 }
1358
1359 static int sdma_v6_0_ring_preempt_ib(struct amdgpu_ring *ring)
1360 {
1361         int i, r = 0;
1362         struct amdgpu_device *adev = ring->adev;
1363         u32 index = 0;
1364         u64 sdma_gfx_preempt;
1365
1366         amdgpu_sdma_get_index_from_ring(ring, &index);
1367         sdma_gfx_preempt =
1368                 sdma_v6_0_get_reg_offset(adev, index, regSDMA0_QUEUE0_PREEMPT);
1369
1370         /* assert preemption condition */
1371         amdgpu_ring_set_preempt_cond_exec(ring, false);
1372
1373         /* emit the trailing fence */
1374         ring->trail_seq += 1;
1375         amdgpu_ring_alloc(ring, 10);
1376         sdma_v6_0_ring_emit_fence(ring, ring->trail_fence_gpu_addr,
1377                                   ring->trail_seq, 0);
1378         amdgpu_ring_commit(ring);
1379
1380         /* assert IB preemption */
1381         WREG32(sdma_gfx_preempt, 1);
1382
1383         /* poll the trailing fence */
1384         for (i = 0; i < adev->usec_timeout; i++) {
1385                 if (ring->trail_seq ==
1386                     le32_to_cpu(*(ring->trail_fence_cpu_addr)))
1387                         break;
1388                 udelay(1);
1389         }
1390
1391         if (i >= adev->usec_timeout) {
1392                 r = -EINVAL;
1393                 DRM_ERROR("ring %d failed to be preempted\n", ring->idx);
1394         }
1395
1396         /* deassert IB preemption */
1397         WREG32(sdma_gfx_preempt, 0);
1398
1399         /* deassert the preemption condition */
1400         amdgpu_ring_set_preempt_cond_exec(ring, true);
1401         return r;
1402 }
1403
1404 static int sdma_v6_0_set_trap_irq_state(struct amdgpu_device *adev,
1405                                         struct amdgpu_irq_src *source,
1406                                         unsigned type,
1407                                         enum amdgpu_interrupt_state state)
1408 {
1409         u32 sdma_cntl;
1410
1411         u32 reg_offset = sdma_v6_0_get_reg_offset(adev, type, regSDMA0_CNTL);
1412
1413         if (!amdgpu_sriov_vf(adev)) {
1414                 sdma_cntl = RREG32(reg_offset);
1415                 sdma_cntl = REG_SET_FIELD(sdma_cntl, SDMA0_CNTL, TRAP_ENABLE,
1416                                 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
1417                 WREG32(reg_offset, sdma_cntl);
1418         }
1419
1420         return 0;
1421 }
1422
1423 static int sdma_v6_0_process_trap_irq(struct amdgpu_device *adev,
1424                                       struct amdgpu_irq_src *source,
1425                                       struct amdgpu_iv_entry *entry)
1426 {
1427         int instances, queue;
1428         uint32_t mes_queue_id = entry->src_data[0];
1429
1430         DRM_DEBUG("IH: SDMA trap\n");
1431
1432         if (adev->enable_mes && (mes_queue_id & AMDGPU_FENCE_MES_QUEUE_FLAG)) {
1433                 struct amdgpu_mes_queue *queue;
1434
1435                 mes_queue_id &= AMDGPU_FENCE_MES_QUEUE_ID_MASK;
1436
1437                 spin_lock(&adev->mes.queue_id_lock);
1438                 queue = idr_find(&adev->mes.queue_id_idr, mes_queue_id);
1439                 if (queue) {
1440                         DRM_DEBUG("process smda queue id = %d\n", mes_queue_id);
1441                         amdgpu_fence_process(queue->ring);
1442                 }
1443                 spin_unlock(&adev->mes.queue_id_lock);
1444                 return 0;
1445         }
1446
1447         queue = entry->ring_id & 0xf;
1448         instances = (entry->ring_id & 0xf0) >> 4;
1449         if (instances > 1) {
1450                 DRM_ERROR("IH: wrong ring_ID detected, as wrong sdma instance\n");
1451                 return -EINVAL;
1452         }
1453
1454         switch (entry->client_id) {
1455         case SOC21_IH_CLIENTID_GFX:
1456                 switch (queue) {
1457                 case 0:
1458                         amdgpu_fence_process(&adev->sdma.instance[instances].ring);
1459                         break;
1460                 default:
1461                         break;
1462                 }
1463                 break;
1464         }
1465         return 0;
1466 }
1467
1468 static int sdma_v6_0_process_illegal_inst_irq(struct amdgpu_device *adev,
1469                                               struct amdgpu_irq_src *source,
1470                                               struct amdgpu_iv_entry *entry)
1471 {
1472         return 0;
1473 }
1474
1475 static int sdma_v6_0_set_clockgating_state(void *handle,
1476                                            enum amd_clockgating_state state)
1477 {
1478         return 0;
1479 }
1480
1481 static int sdma_v6_0_set_powergating_state(void *handle,
1482                                           enum amd_powergating_state state)
1483 {
1484         return 0;
1485 }
1486
1487 static void sdma_v6_0_get_clockgating_state(void *handle, u64 *flags)
1488 {
1489 }
1490
1491 const struct amd_ip_funcs sdma_v6_0_ip_funcs = {
1492         .name = "sdma_v6_0",
1493         .early_init = sdma_v6_0_early_init,
1494         .late_init = NULL,
1495         .sw_init = sdma_v6_0_sw_init,
1496         .sw_fini = sdma_v6_0_sw_fini,
1497         .hw_init = sdma_v6_0_hw_init,
1498         .hw_fini = sdma_v6_0_hw_fini,
1499         .suspend = sdma_v6_0_suspend,
1500         .resume = sdma_v6_0_resume,
1501         .is_idle = sdma_v6_0_is_idle,
1502         .wait_for_idle = sdma_v6_0_wait_for_idle,
1503         .soft_reset = sdma_v6_0_soft_reset,
1504         .check_soft_reset = sdma_v6_0_check_soft_reset,
1505         .set_clockgating_state = sdma_v6_0_set_clockgating_state,
1506         .set_powergating_state = sdma_v6_0_set_powergating_state,
1507         .get_clockgating_state = sdma_v6_0_get_clockgating_state,
1508 };
1509
1510 static const struct amdgpu_ring_funcs sdma_v6_0_ring_funcs = {
1511         .type = AMDGPU_RING_TYPE_SDMA,
1512         .align_mask = 0xf,
1513         .nop = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP),
1514         .support_64bit_ptrs = true,
1515         .secure_submission_supported = true,
1516         .get_rptr = sdma_v6_0_ring_get_rptr,
1517         .get_wptr = sdma_v6_0_ring_get_wptr,
1518         .set_wptr = sdma_v6_0_ring_set_wptr,
1519         .emit_frame_size =
1520                 5 + /* sdma_v6_0_ring_init_cond_exec */
1521                 6 + /* sdma_v6_0_ring_emit_hdp_flush */
1522                 6 + /* sdma_v6_0_ring_emit_pipeline_sync */
1523                 /* sdma_v6_0_ring_emit_vm_flush */
1524                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +
1525                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 6 +
1526                 10 + 10 + 10, /* sdma_v6_0_ring_emit_fence x3 for user fence, vm fence */
1527         .emit_ib_size = 5 + 7 + 6, /* sdma_v6_0_ring_emit_ib */
1528         .emit_ib = sdma_v6_0_ring_emit_ib,
1529         .emit_mem_sync = sdma_v6_0_ring_emit_mem_sync,
1530         .emit_fence = sdma_v6_0_ring_emit_fence,
1531         .emit_pipeline_sync = sdma_v6_0_ring_emit_pipeline_sync,
1532         .emit_vm_flush = sdma_v6_0_ring_emit_vm_flush,
1533         .emit_hdp_flush = sdma_v6_0_ring_emit_hdp_flush,
1534         .test_ring = sdma_v6_0_ring_test_ring,
1535         .test_ib = sdma_v6_0_ring_test_ib,
1536         .insert_nop = sdma_v6_0_ring_insert_nop,
1537         .pad_ib = sdma_v6_0_ring_pad_ib,
1538         .emit_wreg = sdma_v6_0_ring_emit_wreg,
1539         .emit_reg_wait = sdma_v6_0_ring_emit_reg_wait,
1540         .emit_reg_write_reg_wait = sdma_v6_0_ring_emit_reg_write_reg_wait,
1541         .init_cond_exec = sdma_v6_0_ring_init_cond_exec,
1542         .preempt_ib = sdma_v6_0_ring_preempt_ib,
1543 };
1544
1545 static void sdma_v6_0_set_ring_funcs(struct amdgpu_device *adev)
1546 {
1547         int i;
1548
1549         for (i = 0; i < adev->sdma.num_instances; i++) {
1550                 adev->sdma.instance[i].ring.funcs = &sdma_v6_0_ring_funcs;
1551                 adev->sdma.instance[i].ring.me = i;
1552         }
1553 }
1554
1555 static const struct amdgpu_irq_src_funcs sdma_v6_0_trap_irq_funcs = {
1556         .set = sdma_v6_0_set_trap_irq_state,
1557         .process = sdma_v6_0_process_trap_irq,
1558 };
1559
1560 static const struct amdgpu_irq_src_funcs sdma_v6_0_illegal_inst_irq_funcs = {
1561         .process = sdma_v6_0_process_illegal_inst_irq,
1562 };
1563
1564 static void sdma_v6_0_set_irq_funcs(struct amdgpu_device *adev)
1565 {
1566         adev->sdma.trap_irq.num_types = AMDGPU_SDMA_IRQ_INSTANCE0 +
1567                                         adev->sdma.num_instances;
1568         adev->sdma.trap_irq.funcs = &sdma_v6_0_trap_irq_funcs;
1569         adev->sdma.illegal_inst_irq.funcs = &sdma_v6_0_illegal_inst_irq_funcs;
1570 }
1571
1572 /**
1573  * sdma_v6_0_emit_copy_buffer - copy buffer using the sDMA engine
1574  *
1575  * @ib: indirect buffer to fill with commands
1576  * @src_offset: src GPU address
1577  * @dst_offset: dst GPU address
1578  * @byte_count: number of bytes to xfer
1579  * @copy_flags: copy flags for the buffers
1580  *
1581  * Copy GPU buffers using the DMA engine.
1582  * Used by the amdgpu ttm implementation to move pages if
1583  * registered as the asic copy callback.
1584  */
1585 static void sdma_v6_0_emit_copy_buffer(struct amdgpu_ib *ib,
1586                                        uint64_t src_offset,
1587                                        uint64_t dst_offset,
1588                                        uint32_t byte_count,
1589                                        uint32_t copy_flags)
1590 {
1591         ib->ptr[ib->length_dw++] = SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_COPY) |
1592                 SDMA_PKT_COPY_LINEAR_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR) |
1593                 SDMA_PKT_COPY_LINEAR_HEADER_TMZ((copy_flags & AMDGPU_COPY_FLAGS_TMZ) ? 1 : 0);
1594         ib->ptr[ib->length_dw++] = byte_count - 1;
1595         ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */
1596         ib->ptr[ib->length_dw++] = lower_32_bits(src_offset);
1597         ib->ptr[ib->length_dw++] = upper_32_bits(src_offset);
1598         ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset);
1599         ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset);
1600 }
1601
1602 /**
1603  * sdma_v6_0_emit_fill_buffer - fill buffer using the sDMA engine
1604  *
1605  * @ib: indirect buffer to fill
1606  * @src_data: value to write to buffer
1607  * @dst_offset: dst GPU address
1608  * @byte_count: number of bytes to xfer
1609  *
1610  * Fill GPU buffers using the DMA engine.
1611  */
1612 static void sdma_v6_0_emit_fill_buffer(struct amdgpu_ib *ib,
1613                                        uint32_t src_data,
1614                                        uint64_t dst_offset,
1615                                        uint32_t byte_count)
1616 {
1617         ib->ptr[ib->length_dw++] = SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_CONST_FILL);
1618         ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset);
1619         ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset);
1620         ib->ptr[ib->length_dw++] = src_data;
1621         ib->ptr[ib->length_dw++] = byte_count - 1;
1622 }
1623
1624 static const struct amdgpu_buffer_funcs sdma_v6_0_buffer_funcs = {
1625         .copy_max_bytes = 0x400000,
1626         .copy_num_dw = 7,
1627         .emit_copy_buffer = sdma_v6_0_emit_copy_buffer,
1628
1629         .fill_max_bytes = 0x400000,
1630         .fill_num_dw = 5,
1631         .emit_fill_buffer = sdma_v6_0_emit_fill_buffer,
1632 };
1633
1634 static void sdma_v6_0_set_buffer_funcs(struct amdgpu_device *adev)
1635 {
1636         adev->mman.buffer_funcs = &sdma_v6_0_buffer_funcs;
1637         adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring;
1638 }
1639
1640 static const struct amdgpu_vm_pte_funcs sdma_v6_0_vm_pte_funcs = {
1641         .copy_pte_num_dw = 7,
1642         .copy_pte = sdma_v6_0_vm_copy_pte,
1643         .write_pte = sdma_v6_0_vm_write_pte,
1644         .set_pte_pde = sdma_v6_0_vm_set_pte_pde,
1645 };
1646
1647 static void sdma_v6_0_set_vm_pte_funcs(struct amdgpu_device *adev)
1648 {
1649         unsigned i;
1650
1651         adev->vm_manager.vm_pte_funcs = &sdma_v6_0_vm_pte_funcs;
1652         for (i = 0; i < adev->sdma.num_instances; i++) {
1653                 adev->vm_manager.vm_pte_scheds[i] =
1654                         &adev->sdma.instance[i].ring.sched;
1655         }
1656         adev->vm_manager.vm_pte_num_scheds = adev->sdma.num_instances;
1657 }
1658
1659 const struct amdgpu_ip_block_version sdma_v6_0_ip_block = {
1660         .type = AMD_IP_BLOCK_TYPE_SDMA,
1661         .major = 6,
1662         .minor = 0,
1663         .rev = 0,
1664         .funcs = &sdma_v6_0_ip_funcs,
1665 };