drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c

   1 /*
   2  * Copyright 2021 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23 #include <linux/delay.h>
  24 #include <linux/kernel.h>
  25 #include <linux/firmware.h>
  26 #include <linux/module.h>
  27 #include <linux/pci.h>
  28 #include "amdgpu.h"
  29 #include "amdgpu_gfx.h"
  30 #include "amdgpu_psp.h"
  31 #include "amdgpu_smu.h"
  32 #include "amdgpu_atomfirmware.h"
  33 #include "imu_v11_0.h"
  34 #include "soc21.h"
  35 #include "nvd.h"
  36
  37 #include "gc/gc_11_0_0_offset.h"
  38 #include "gc/gc_11_0_0_sh_mask.h"
  39 #include "smuio/smuio_13_0_6_offset.h"
  40 #include "smuio/smuio_13_0_6_sh_mask.h"
  41 #include "navi10_enum.h"
  42 #include "ivsrcid/gfx/irqsrcs_gfx_11_0_0.h"
  43
  44 #include "soc15.h"
  45 #include "soc15d.h"
  46 #include "clearstate_gfx11.h"
  47 #include "v11_structs.h"
  48 #include "gfx_v11_0.h"
  49 #include "gfx_v11_0_3.h"
  50 #include "nbio_v4_3.h"
  51 #include "mes_v11_0.h"
  52
  53 #define GFX11_NUM_GFX_RINGS             1
  54 #define GFX11_MEC_HPD_SIZE      2048
  55
  56 #define RLCG_UCODE_LOADING_START_ADDRESS        0x00002000L
  57 #define RLC_PG_DELAY_3_DEFAULT_GC_11_0_1        0x1388
  58
  59 #define regCGTT_WD_CLK_CTRL             0x5086
  60 #define regCGTT_WD_CLK_CTRL_BASE_IDX    1
  61 #define regRLC_RLCS_BOOTLOAD_STATUS_gc_11_0_1   0x4e7e
  62 #define regRLC_RLCS_BOOTLOAD_STATUS_gc_11_0_1_BASE_IDX  1
  63
  64 MODULE_FIRMWARE("amdgpu/gc_11_0_0_pfp.bin");
  65 MODULE_FIRMWARE("amdgpu/gc_11_0_0_me.bin");
  66 MODULE_FIRMWARE("amdgpu/gc_11_0_0_mec.bin");
  67 MODULE_FIRMWARE("amdgpu/gc_11_0_0_rlc.bin");
  68 MODULE_FIRMWARE("amdgpu/gc_11_0_0_toc.bin");
  69 MODULE_FIRMWARE("amdgpu/gc_11_0_1_pfp.bin");
  70 MODULE_FIRMWARE("amdgpu/gc_11_0_1_me.bin");
  71 MODULE_FIRMWARE("amdgpu/gc_11_0_1_mec.bin");
  72 MODULE_FIRMWARE("amdgpu/gc_11_0_1_rlc.bin");
  73 MODULE_FIRMWARE("amdgpu/gc_11_0_2_pfp.bin");
  74 MODULE_FIRMWARE("amdgpu/gc_11_0_2_me.bin");
  75 MODULE_FIRMWARE("amdgpu/gc_11_0_2_mec.bin");
  76 MODULE_FIRMWARE("amdgpu/gc_11_0_2_rlc.bin");
  77 MODULE_FIRMWARE("amdgpu/gc_11_0_3_pfp.bin");
  78 MODULE_FIRMWARE("amdgpu/gc_11_0_3_me.bin");
  79 MODULE_FIRMWARE("amdgpu/gc_11_0_3_mec.bin");
  80 MODULE_FIRMWARE("amdgpu/gc_11_0_3_rlc.bin");
  81 MODULE_FIRMWARE("amdgpu/gc_11_0_4_pfp.bin");
  82 MODULE_FIRMWARE("amdgpu/gc_11_0_4_me.bin");
  83 MODULE_FIRMWARE("amdgpu/gc_11_0_4_mec.bin");
  84 MODULE_FIRMWARE("amdgpu/gc_11_0_4_rlc.bin");
  85
  86 static const struct soc15_reg_golden golden_settings_gc_11_0_1[] =
  87 {
  88         SOC15_REG_GOLDEN_VALUE(GC, 0, regCGTT_GS_NGG_CLK_CTRL, 0x9fff8fff, 0x00000010),
  89         SOC15_REG_GOLDEN_VALUE(GC, 0, regCGTT_WD_CLK_CTRL, 0xffff8fff, 0x00000010),
  90         SOC15_REG_GOLDEN_VALUE(GC, 0, regCPF_GCR_CNTL, 0x0007ffff, 0x0000c200),
  91         SOC15_REG_GOLDEN_VALUE(GC, 0, regGL2C_CTRL3, 0xffff001b, 0x00f01988),
  92         SOC15_REG_GOLDEN_VALUE(GC, 0, regPA_CL_ENHANCE, 0xf0ffffff, 0x00880007),
  93         SOC15_REG_GOLDEN_VALUE(GC, 0, regPA_SC_ENHANCE_3, 0xfffffffd, 0x00000008),
  94         SOC15_REG_GOLDEN_VALUE(GC, 0, regPA_SC_VRS_SURFACE_CNTL_1, 0xfff891ff, 0x55480100),
  95         SOC15_REG_GOLDEN_VALUE(GC, 0, regTA_CNTL_AUX, 0xf7f7ffff, 0x01030000),
  96         SOC15_REG_GOLDEN_VALUE(GC, 0, regTCP_CNTL2, 0xfcffffff, 0x0000000a)
  97 };
  98
  99 #define DEFAULT_SH_MEM_CONFIG \
 100         ((SH_MEM_ADDRESS_MODE_64 << SH_MEM_CONFIG__ADDRESS_MODE__SHIFT) | \
 101          (SH_MEM_ALIGNMENT_MODE_UNALIGNED << SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT) | \
 102          (3 << SH_MEM_CONFIG__INITIAL_INST_PREFETCH__SHIFT))
 103
 104 static void gfx_v11_0_disable_gpa_mode(struct amdgpu_device *adev);
 105 static void gfx_v11_0_set_ring_funcs(struct amdgpu_device *adev);
 106 static void gfx_v11_0_set_irq_funcs(struct amdgpu_device *adev);
 107 static void gfx_v11_0_set_gds_init(struct amdgpu_device *adev);
 108 static void gfx_v11_0_set_rlc_funcs(struct amdgpu_device *adev);
 109 static void gfx_v11_0_set_mqd_funcs(struct amdgpu_device *adev);
 110 static void gfx_v11_0_set_imu_funcs(struct amdgpu_device *adev);
 111 static int gfx_v11_0_get_cu_info(struct amdgpu_device *adev,
 112                                  struct amdgpu_cu_info *cu_info);
 113 static uint64_t gfx_v11_0_get_gpu_clock_counter(struct amdgpu_device *adev);
 114 static void gfx_v11_0_select_se_sh(struct amdgpu_device *adev, u32 se_num,
 115                                    u32 sh_num, u32 instance);
 116 static u32 gfx_v11_0_get_wgp_active_bitmap_per_sh(struct amdgpu_device *adev);
 117
 118 static void gfx_v11_0_ring_emit_de_meta(struct amdgpu_ring *ring, bool resume);
 119 static void gfx_v11_0_ring_emit_frame_cntl(struct amdgpu_ring *ring, bool start, bool secure);
 120 static void gfx_v11_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg,
 121                                      uint32_t val);
 122 static int gfx_v11_0_wait_for_rlc_autoload_complete(struct amdgpu_device *adev);
 123 static void gfx_v11_0_ring_invalidate_tlbs(struct amdgpu_ring *ring,
 124                                            uint16_t pasid, uint32_t flush_type,
 125                                            bool all_hub, uint8_t dst_sel);
 126 static void gfx_v11_0_set_safe_mode(struct amdgpu_device *adev);
 127 static void gfx_v11_0_unset_safe_mode(struct amdgpu_device *adev);
 128 static void gfx_v11_0_update_perf_clk(struct amdgpu_device *adev,
 129                                       bool enable);
 130
 131 static void gfx11_kiq_set_resources(struct amdgpu_ring *kiq_ring, uint64_t queue_mask)
 132 {
 133         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_RESOURCES, 6));
 134         amdgpu_ring_write(kiq_ring, PACKET3_SET_RESOURCES_VMID_MASK(0) |
 135                           PACKET3_SET_RESOURCES_QUEUE_TYPE(0)); /* vmid_mask:0 queue_type:0 (KIQ) */
 136         amdgpu_ring_write(kiq_ring, lower_32_bits(queue_mask)); /* queue mask lo */
 137         amdgpu_ring_write(kiq_ring, upper_32_bits(queue_mask)); /* queue mask hi */
 138         amdgpu_ring_write(kiq_ring, 0); /* gws mask lo */
 139         amdgpu_ring_write(kiq_ring, 0); /* gws mask hi */
 140         amdgpu_ring_write(kiq_ring, 0); /* oac mask */
 141         amdgpu_ring_write(kiq_ring, 0); /* gds heap base:0, gds heap size:0 */
 142 }
 143
 144 static void gfx11_kiq_map_queues(struct amdgpu_ring *kiq_ring,
 145                                  struct amdgpu_ring *ring)
 146 {
 147         uint64_t mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj);
 148         uint64_t wptr_addr = ring->wptr_gpu_addr;
 149         uint32_t me = 0, eng_sel = 0;
 150
 151         switch (ring->funcs->type) {
 152         case AMDGPU_RING_TYPE_COMPUTE:
 153                 me = 1;
 154                 eng_sel = 0;
 155                 break;
 156         case AMDGPU_RING_TYPE_GFX:
 157                 me = 0;
 158                 eng_sel = 4;
 159                 break;
 160         case AMDGPU_RING_TYPE_MES:
 161                 me = 2;
 162                 eng_sel = 5;
 163                 break;
 164         default:
 165                 WARN_ON(1);
 166         }
 167
 168         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5));
 169         /* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/
 170         amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
 171                           PACKET3_MAP_QUEUES_QUEUE_SEL(0) | /* Queue_Sel */
 172                           PACKET3_MAP_QUEUES_VMID(0) | /* VMID */
 173                           PACKET3_MAP_QUEUES_QUEUE(ring->queue) |
 174                           PACKET3_MAP_QUEUES_PIPE(ring->pipe) |
 175                           PACKET3_MAP_QUEUES_ME((me)) |
 176                           PACKET3_MAP_QUEUES_QUEUE_TYPE(0) | /*queue_type: normal compute queue */
 177                           PACKET3_MAP_QUEUES_ALLOC_FORMAT(0) | /* alloc format: all_on_one_pipe */
 178                           PACKET3_MAP_QUEUES_ENGINE_SEL(eng_sel) |
 179                           PACKET3_MAP_QUEUES_NUM_QUEUES(1)); /* num_queues: must be 1 */
 180         amdgpu_ring_write(kiq_ring, PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index));
 181         amdgpu_ring_write(kiq_ring, lower_32_bits(mqd_addr));
 182         amdgpu_ring_write(kiq_ring, upper_32_bits(mqd_addr));
 183         amdgpu_ring_write(kiq_ring, lower_32_bits(wptr_addr));
 184         amdgpu_ring_write(kiq_ring, upper_32_bits(wptr_addr));
 185 }
 186
 187 static void gfx11_kiq_unmap_queues(struct amdgpu_ring *kiq_ring,
 188                                    struct amdgpu_ring *ring,
 189                                    enum amdgpu_unmap_queues_action action,
 190                                    u64 gpu_addr, u64 seq)
 191 {
 192         struct amdgpu_device *adev = kiq_ring->adev;
 193         uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
 194
 195         if (adev->enable_mes && !adev->gfx.kiq.ring.sched.ready) {
 196                 amdgpu_mes_unmap_legacy_queue(adev, ring, action, gpu_addr, seq);
 197                 return;
 198         }
 199
 200         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4));
 201         amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
 202                           PACKET3_UNMAP_QUEUES_ACTION(action) |
 203                           PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) |
 204                           PACKET3_UNMAP_QUEUES_ENGINE_SEL(eng_sel) |
 205                           PACKET3_UNMAP_QUEUES_NUM_QUEUES(1));
 206         amdgpu_ring_write(kiq_ring,
 207                   PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index));
 208
 209         if (action == PREEMPT_QUEUES_NO_UNMAP) {
 210                 amdgpu_ring_write(kiq_ring, lower_32_bits(gpu_addr));
 211                 amdgpu_ring_write(kiq_ring, upper_32_bits(gpu_addr));
 212                 amdgpu_ring_write(kiq_ring, seq);
 213         } else {
 214                 amdgpu_ring_write(kiq_ring, 0);
 215                 amdgpu_ring_write(kiq_ring, 0);
 216                 amdgpu_ring_write(kiq_ring, 0);
 217         }
 218 }
 219
 220 static void gfx11_kiq_query_status(struct amdgpu_ring *kiq_ring,
 221                                    struct amdgpu_ring *ring,
 222                                    u64 addr,
 223                                    u64 seq)
 224 {
 225         uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
 226
 227         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_QUERY_STATUS, 5));
 228         amdgpu_ring_write(kiq_ring,
 229                           PACKET3_QUERY_STATUS_CONTEXT_ID(0) |
 230                           PACKET3_QUERY_STATUS_INTERRUPT_SEL(0) |
 231                           PACKET3_QUERY_STATUS_COMMAND(2));
 232         amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
 233                           PACKET3_QUERY_STATUS_DOORBELL_OFFSET(ring->doorbell_index) |
 234                           PACKET3_QUERY_STATUS_ENG_SEL(eng_sel));
 235         amdgpu_ring_write(kiq_ring, lower_32_bits(addr));
 236         amdgpu_ring_write(kiq_ring, upper_32_bits(addr));
 237         amdgpu_ring_write(kiq_ring, lower_32_bits(seq));
 238         amdgpu_ring_write(kiq_ring, upper_32_bits(seq));
 239 }
 240
 241 static void gfx11_kiq_invalidate_tlbs(struct amdgpu_ring *kiq_ring,
 242                                 uint16_t pasid, uint32_t flush_type,
 243                                 bool all_hub)
 244 {
 245         gfx_v11_0_ring_invalidate_tlbs(kiq_ring, pasid, flush_type, all_hub, 1);
 246 }
 247
 248 static const struct kiq_pm4_funcs gfx_v11_0_kiq_pm4_funcs = {
 249         .kiq_set_resources = gfx11_kiq_set_resources,
 250         .kiq_map_queues = gfx11_kiq_map_queues,
 251         .kiq_unmap_queues = gfx11_kiq_unmap_queues,
 252         .kiq_query_status = gfx11_kiq_query_status,
 253         .kiq_invalidate_tlbs = gfx11_kiq_invalidate_tlbs,
 254         .set_resources_size = 8,
 255         .map_queues_size = 7,
 256         .unmap_queues_size = 6,
 257         .query_status_size = 7,
 258         .invalidate_tlbs_size = 2,
 259 };
 260
 261 static void gfx_v11_0_set_kiq_pm4_funcs(struct amdgpu_device *adev)
 262 {
 263         adev->gfx.kiq.pmf = &gfx_v11_0_kiq_pm4_funcs;
 264 }
 265
 266 static void gfx_v11_0_init_golden_registers(struct amdgpu_device *adev)
 267 {
 268         switch (adev->ip_versions[GC_HWIP][0]) {
 269         case IP_VERSION(11, 0, 1):
 270         case IP_VERSION(11, 0, 4):
 271                 soc15_program_register_sequence(adev,
 272                                                 golden_settings_gc_11_0_1,
 273                                                 (const u32)ARRAY_SIZE(golden_settings_gc_11_0_1));
 274                 break;
 275         default:
 276                 break;
 277         }
 278 }
 279
 280 static void gfx_v11_0_write_data_to_reg(struct amdgpu_ring *ring, int eng_sel,
 281                                        bool wc, uint32_t reg, uint32_t val)
 282 {
 283         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
 284         amdgpu_ring_write(ring, WRITE_DATA_ENGINE_SEL(eng_sel) |
 285                           WRITE_DATA_DST_SEL(0) | (wc ? WR_CONFIRM : 0));
 286         amdgpu_ring_write(ring, reg);
 287         amdgpu_ring_write(ring, 0);
 288         amdgpu_ring_write(ring, val);
 289 }
 290
 291 static void gfx_v11_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel,
 292                                   int mem_space, int opt, uint32_t addr0,
 293                                   uint32_t addr1, uint32_t ref, uint32_t mask,
 294                                   uint32_t inv)
 295 {
 296         amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
 297         amdgpu_ring_write(ring,
 298                           /* memory (1) or register (0) */
 299                           (WAIT_REG_MEM_MEM_SPACE(mem_space) |
 300                            WAIT_REG_MEM_OPERATION(opt) | /* wait */
 301                            WAIT_REG_MEM_FUNCTION(3) |  /* equal */
 302                            WAIT_REG_MEM_ENGINE(eng_sel)));
 303
 304         if (mem_space)
 305                 BUG_ON(addr0 & 0x3); /* Dword align */
 306         amdgpu_ring_write(ring, addr0);
 307         amdgpu_ring_write(ring, addr1);
 308         amdgpu_ring_write(ring, ref);
 309         amdgpu_ring_write(ring, mask);
 310         amdgpu_ring_write(ring, inv); /* poll interval */
 311 }
 312
 313 static int gfx_v11_0_ring_test_ring(struct amdgpu_ring *ring)
 314 {
 315         struct amdgpu_device *adev = ring->adev;
 316         uint32_t scratch = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG0);
 317         uint32_t tmp = 0;
 318         unsigned i;
 319         int r;
 320
 321         WREG32(scratch, 0xCAFEDEAD);
 322         r = amdgpu_ring_alloc(ring, 5);
 323         if (r) {
 324                 DRM_ERROR("amdgpu: cp failed to lock ring %d (%d).\n",
 325                           ring->idx, r);
 326                 return r;
 327         }
 328
 329         if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ) {
 330                 gfx_v11_0_ring_emit_wreg(ring, scratch, 0xDEADBEEF);
 331         } else {
 332                 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
 333                 amdgpu_ring_write(ring, scratch -
 334                                   PACKET3_SET_UCONFIG_REG_START);
 335                 amdgpu_ring_write(ring, 0xDEADBEEF);
 336         }
 337         amdgpu_ring_commit(ring);
 338
 339         for (i = 0; i < adev->usec_timeout; i++) {
 340                 tmp = RREG32(scratch);
 341                 if (tmp == 0xDEADBEEF)
 342                         break;
 343                 if (amdgpu_emu_mode == 1)
 344                         msleep(1);
 345                 else
 346                         udelay(1);
 347         }
 348
 349         if (i >= adev->usec_timeout)
 350                 r = -ETIMEDOUT;
 351         return r;
 352 }
 353
 354 static int gfx_v11_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 355 {
 356         struct amdgpu_device *adev = ring->adev;
 357         struct amdgpu_ib ib;
 358         struct dma_fence *f = NULL;
 359         unsigned index;
 360         uint64_t gpu_addr;
 361         volatile uint32_t *cpu_ptr;
 362         long r;
 363
 364         /* MES KIQ fw hasn't indirect buffer support for now */
 365         if (adev->enable_mes_kiq &&
 366             ring->funcs->type == AMDGPU_RING_TYPE_KIQ)
 367                 return 0;
 368
 369         memset(&ib, 0, sizeof(ib));
 370
 371         if (ring->is_mes_queue) {
 372                 uint32_t padding, offset;
 373
 374                 offset = amdgpu_mes_ctx_get_offs(ring, AMDGPU_MES_CTX_IB_OFFS);
 375                 padding = amdgpu_mes_ctx_get_offs(ring,
 376                                                   AMDGPU_MES_CTX_PADDING_OFFS);
 377
 378                 ib.gpu_addr = amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset);
 379                 ib.ptr = amdgpu_mes_ctx_get_offs_cpu_addr(ring, offset);
 380
 381                 gpu_addr = amdgpu_mes_ctx_get_offs_gpu_addr(ring, padding);
 382                 cpu_ptr = amdgpu_mes_ctx_get_offs_cpu_addr(ring, padding);
 383                 *cpu_ptr = cpu_to_le32(0xCAFEDEAD);
 384         } else {
 385                 r = amdgpu_device_wb_get(adev, &index);
 386                 if (r)
 387                         return r;
 388
 389                 gpu_addr = adev->wb.gpu_addr + (index * 4);
 390                 adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD);
 391                 cpu_ptr = &adev->wb.wb[index];
 392
 393                 r = amdgpu_ib_get(adev, NULL, 16, AMDGPU_IB_POOL_DIRECT, &ib);
 394                 if (r) {
 395                         DRM_ERROR("amdgpu: failed to get ib (%ld).\n", r);
 396                         goto err1;
 397                 }
 398         }
 399
 400         ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3);
 401         ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM;
 402         ib.ptr[2] = lower_32_bits(gpu_addr);
 403         ib.ptr[3] = upper_32_bits(gpu_addr);
 404         ib.ptr[4] = 0xDEADBEEF;
 405         ib.length_dw = 5;
 406
 407         r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
 408         if (r)
 409                 goto err2;
 410
 411         r = dma_fence_wait_timeout(f, false, timeout);
 412         if (r == 0) {
 413                 r = -ETIMEDOUT;
 414                 goto err2;
 415         } else if (r < 0) {
 416                 goto err2;
 417         }
 418
 419         if (le32_to_cpu(*cpu_ptr) == 0xDEADBEEF)
 420                 r = 0;
 421         else
 422                 r = -EINVAL;
 423 err2:
 424         if (!ring->is_mes_queue)
 425                 amdgpu_ib_free(adev, &ib, NULL);
 426         dma_fence_put(f);
 427 err1:
 428         if (!ring->is_mes_queue)
 429                 amdgpu_device_wb_free(adev, index);
 430         return r;
 431 }
 432
 433 static void gfx_v11_0_free_microcode(struct amdgpu_device *adev)
 434 {
 435         amdgpu_ucode_release(&adev->gfx.pfp_fw);
 436         amdgpu_ucode_release(&adev->gfx.me_fw);
 437         amdgpu_ucode_release(&adev->gfx.rlc_fw);
 438         amdgpu_ucode_release(&adev->gfx.mec_fw);
 439
 440         kfree(adev->gfx.rlc.register_list_format);
 441 }
 442
 443 static int gfx_v11_0_init_toc_microcode(struct amdgpu_device *adev, const char *ucode_prefix)
 444 {
 445         const struct psp_firmware_header_v1_0 *toc_hdr;
 446         int err = 0;
 447         char fw_name[40];
 448
 449         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_toc.bin", ucode_prefix);
 450         err = amdgpu_ucode_request(adev, &adev->psp.toc_fw, fw_name);
 451         if (err)
 452                 goto out;
 453
 454         toc_hdr = (const struct psp_firmware_header_v1_0 *)adev->psp.toc_fw->data;
 455         adev->psp.toc.fw_version = le32_to_cpu(toc_hdr->header.ucode_version);
 456         adev->psp.toc.feature_version = le32_to_cpu(toc_hdr->sos.fw_version);
 457         adev->psp.toc.size_bytes = le32_to_cpu(toc_hdr->header.ucode_size_bytes);
 458         adev->psp.toc.start_addr = (uint8_t *)toc_hdr +
 459                                 le32_to_cpu(toc_hdr->header.ucode_array_offset_bytes);
 460         return 0;
 461 out:
 462         amdgpu_ucode_release(&adev->psp.toc_fw);
 463         return err;
 464 }
 465
 466 static int gfx_v11_0_init_microcode(struct amdgpu_device *adev)
 467 {
 468         char fw_name[40];
 469         char ucode_prefix[30];
 470         int err;
 471         const struct rlc_firmware_header_v2_0 *rlc_hdr;
 472         uint16_t version_major;
 473         uint16_t version_minor;
 474
 475         DRM_DEBUG("\n");
 476
 477         amdgpu_ucode_ip_version_decode(adev, GC_HWIP, ucode_prefix, sizeof(ucode_prefix));
 478
 479         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_pfp.bin", ucode_prefix);
 480         err = amdgpu_ucode_request(adev, &adev->gfx.pfp_fw, fw_name);
 481         if (err)
 482                 goto out;
 483         /* check pfp fw hdr version to decide if enable rs64 for gfx11.*/
 484         adev->gfx.rs64_enable = amdgpu_ucode_hdr_version(
 485                                 (union amdgpu_firmware_header *)
 486                                 adev->gfx.pfp_fw->data, 2, 0);
 487         if (adev->gfx.rs64_enable) {
 488                 dev_info(adev->dev, "CP RS64 enable\n");
 489                 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_PFP);
 490                 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_PFP_P0_STACK);
 491                 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_PFP_P1_STACK);
 492         } else {
 493                 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_PFP);
 494         }
 495
 496         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_me.bin", ucode_prefix);
 497         err = amdgpu_ucode_request(adev, &adev->gfx.me_fw, fw_name);
 498         if (err)
 499                 goto out;
 500         if (adev->gfx.rs64_enable) {
 501                 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_ME);
 502                 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_ME_P0_STACK);
 503                 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_ME_P1_STACK);
 504         } else {
 505                 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_ME);
 506         }
 507
 508         if (!amdgpu_sriov_vf(adev)) {
 509                 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_rlc.bin", ucode_prefix);
 510                 err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw, fw_name);
 511                 if (err)
 512                         goto out;
 513                 rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
 514                 version_major = le16_to_cpu(rlc_hdr->header.header_version_major);
 515                 version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor);
 516                 err = amdgpu_gfx_rlc_init_microcode(adev, version_major, version_minor);
 517                 if (err)
 518                         goto out;
 519         }
 520
 521         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec.bin", ucode_prefix);
 522         err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw, fw_name);
 523         if (err)
 524                 goto out;
 525         if (adev->gfx.rs64_enable) {
 526                 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC);
 527                 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P0_STACK);
 528                 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P1_STACK);
 529                 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P2_STACK);
 530                 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P3_STACK);
 531         } else {
 532                 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC1);
 533                 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC1_JT);
 534         }
 535
 536         if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO)
 537                 err = gfx_v11_0_init_toc_microcode(adev, ucode_prefix);
 538
 539         /* only one MEC for gfx 11.0.0. */
 540         adev->gfx.mec2_fw = NULL;
 541
 542 out:
 543         if (err) {
 544                 amdgpu_ucode_release(&adev->gfx.pfp_fw);
 545                 amdgpu_ucode_release(&adev->gfx.me_fw);
 546                 amdgpu_ucode_release(&adev->gfx.rlc_fw);
 547                 amdgpu_ucode_release(&adev->gfx.mec_fw);
 548         }
 549
 550         return err;
 551 }
 552
 553 static u32 gfx_v11_0_get_csb_size(struct amdgpu_device *adev)
 554 {
 555         u32 count = 0;
 556         const struct cs_section_def *sect = NULL;
 557         const struct cs_extent_def *ext = NULL;
 558
 559         /* begin clear state */
 560         count += 2;
 561         /* context control state */
 562         count += 3;
 563
 564         for (sect = gfx11_cs_data; sect->section != NULL; ++sect) {
 565                 for (ext = sect->section; ext->extent != NULL; ++ext) {
 566                         if (sect->id == SECT_CONTEXT)
 567                                 count += 2 + ext->reg_count;
 568                         else
 569                                 return 0;
 570                 }
 571         }
 572
 573         /* set PA_SC_TILE_STEERING_OVERRIDE */
 574         count += 3;
 575         /* end clear state */
 576         count += 2;
 577         /* clear state */
 578         count += 2;
 579
 580         return count;
 581 }
 582
 583 static void gfx_v11_0_get_csb_buffer(struct amdgpu_device *adev,
 584                                     volatile u32 *buffer)
 585 {
 586         u32 count = 0, i;
 587         const struct cs_section_def *sect = NULL;
 588         const struct cs_extent_def *ext = NULL;
 589         int ctx_reg_offset;
 590
 591         if (adev->gfx.rlc.cs_data == NULL)
 592                 return;
 593         if (buffer == NULL)
 594                 return;
 595
 596         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
 597         buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
 598
 599         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
 600         buffer[count++] = cpu_to_le32(0x80000000);
 601         buffer[count++] = cpu_to_le32(0x80000000);
 602
 603         for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) {
 604                 for (ext = sect->section; ext->extent != NULL; ++ext) {
 605                         if (sect->id == SECT_CONTEXT) {
 606                                 buffer[count++] =
 607                                         cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
 608                                 buffer[count++] = cpu_to_le32(ext->reg_index -
 609                                                 PACKET3_SET_CONTEXT_REG_START);
 610                                 for (i = 0; i < ext->reg_count; i++)
 611                                         buffer[count++] = cpu_to_le32(ext->extent[i]);
 612                         } else {
 613                                 return;
 614                         }
 615                 }
 616         }
 617
 618         ctx_reg_offset =
 619                 SOC15_REG_OFFSET(GC, 0, regPA_SC_TILE_STEERING_OVERRIDE) - PACKET3_SET_CONTEXT_REG_START;
 620         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 1));
 621         buffer[count++] = cpu_to_le32(ctx_reg_offset);
 622         buffer[count++] = cpu_to_le32(adev->gfx.config.pa_sc_tile_steering_override);
 623
 624         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
 625         buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
 626
 627         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
 628         buffer[count++] = cpu_to_le32(0);
 629 }
 630
 631 static void gfx_v11_0_rlc_fini(struct amdgpu_device *adev)
 632 {
 633         /* clear state block */
 634         amdgpu_bo_free_kernel(&adev->gfx.rlc.clear_state_obj,
 635                         &adev->gfx.rlc.clear_state_gpu_addr,
 636                         (void **)&adev->gfx.rlc.cs_ptr);
 637
 638         /* jump table block */
 639         amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj,
 640                         &adev->gfx.rlc.cp_table_gpu_addr,
 641                         (void **)&adev->gfx.rlc.cp_table_ptr);
 642 }
 643
 644 static void gfx_v11_0_init_rlcg_reg_access_ctrl(struct amdgpu_device *adev)
 645 {
 646         struct amdgpu_rlcg_reg_access_ctrl *reg_access_ctrl;
 647
 648         reg_access_ctrl = &adev->gfx.rlc.reg_access_ctrl;
 649         reg_access_ctrl->scratch_reg0 = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG0);
 650         reg_access_ctrl->scratch_reg1 = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG1);
 651         reg_access_ctrl->scratch_reg2 = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG2);
 652         reg_access_ctrl->scratch_reg3 = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG3);
 653         reg_access_ctrl->grbm_cntl = SOC15_REG_OFFSET(GC, 0, regGRBM_GFX_CNTL);
 654         reg_access_ctrl->grbm_idx = SOC15_REG_OFFSET(GC, 0, regGRBM_GFX_INDEX);
 655         reg_access_ctrl->spare_int = SOC15_REG_OFFSET(GC, 0, regRLC_SPARE_INT_0);
 656         adev->gfx.rlc.rlcg_reg_access_supported = true;
 657 }
 658
 659 static int gfx_v11_0_rlc_init(struct amdgpu_device *adev)
 660 {
 661         const struct cs_section_def *cs_data;
 662         int r;
 663
 664         adev->gfx.rlc.cs_data = gfx11_cs_data;
 665
 666         cs_data = adev->gfx.rlc.cs_data;
 667
 668         if (cs_data) {
 669                 /* init clear state block */
 670                 r = amdgpu_gfx_rlc_init_csb(adev);
 671                 if (r)
 672                         return r;
 673         }
 674
 675         /* init spm vmid with 0xf */
 676         if (adev->gfx.rlc.funcs->update_spm_vmid)
 677                 adev->gfx.rlc.funcs->update_spm_vmid(adev, 0xf);
 678
 679         return 0;
 680 }
 681
 682 static void gfx_v11_0_mec_fini(struct amdgpu_device *adev)
 683 {
 684         amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL);
 685         amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_obj, NULL, NULL);
 686         amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_data_obj, NULL, NULL);
 687 }
 688
 689 static void gfx_v11_0_me_init(struct amdgpu_device *adev)
 690 {
 691         bitmap_zero(adev->gfx.me.queue_bitmap, AMDGPU_MAX_GFX_QUEUES);
 692
 693         amdgpu_gfx_graphics_queue_acquire(adev);
 694 }
 695
 696 static int gfx_v11_0_mec_init(struct amdgpu_device *adev)
 697 {
 698         int r;
 699         u32 *hpd;
 700         size_t mec_hpd_size;
 701
 702         bitmap_zero(adev->gfx.mec.queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
 703
 704         /* take ownership of the relevant compute queues */
 705         amdgpu_gfx_compute_queue_acquire(adev);
 706         mec_hpd_size = adev->gfx.num_compute_rings * GFX11_MEC_HPD_SIZE;
 707
 708         if (mec_hpd_size) {
 709                 r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE,
 710                                               AMDGPU_GEM_DOMAIN_GTT,
 711                                               &adev->gfx.mec.hpd_eop_obj,
 712                                               &adev->gfx.mec.hpd_eop_gpu_addr,
 713                                               (void **)&hpd);
 714                 if (r) {
 715                         dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r);
 716                         gfx_v11_0_mec_fini(adev);
 717                         return r;
 718                 }
 719
 720                 memset(hpd, 0, mec_hpd_size);
 721
 722                 amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj);
 723                 amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj);
 724         }
 725
 726         return 0;
 727 }
 728
 729 static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t wave, uint32_t address)
 730 {
 731         WREG32_SOC15(GC, 0, regSQ_IND_INDEX,
 732                 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
 733                 (address << SQ_IND_INDEX__INDEX__SHIFT));
 734         return RREG32_SOC15(GC, 0, regSQ_IND_DATA);
 735 }
 736
 737 static void wave_read_regs(struct amdgpu_device *adev, uint32_t wave,
 738                            uint32_t thread, uint32_t regno,
 739                            uint32_t num, uint32_t *out)
 740 {
 741         WREG32_SOC15(GC, 0, regSQ_IND_INDEX,
 742                 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
 743                 (regno << SQ_IND_INDEX__INDEX__SHIFT) |
 744                 (thread << SQ_IND_INDEX__WORKITEM_ID__SHIFT) |
 745                 (SQ_IND_INDEX__AUTO_INCR_MASK));
 746         while (num--)
 747                 *(out++) = RREG32_SOC15(GC, 0, regSQ_IND_DATA);
 748 }
 749
 750 static void gfx_v11_0_read_wave_data(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields)
 751 {
 752         /* in gfx11 the SIMD_ID is specified as part of the INSTANCE
 753          * field when performing a select_se_sh so it should be
 754          * zero here */
 755         WARN_ON(simd != 0);
 756
 757         /* type 3 wave data */
 758         dst[(*no_fields)++] = 3;
 759         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_STATUS);
 760         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_PC_LO);
 761         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_PC_HI);
 762         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_EXEC_LO);
 763         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_EXEC_HI);
 764         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_HW_ID1);
 765         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_HW_ID2);
 766         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_GPR_ALLOC);
 767         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_LDS_ALLOC);
 768         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_TRAPSTS);
 769         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_IB_STS);
 770         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_IB_STS2);
 771         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_IB_DBG1);
 772         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_M0);
 773         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_MODE);
 774 }
 775
 776 static void gfx_v11_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t simd,
 777                                      uint32_t wave, uint32_t start,
 778                                      uint32_t size, uint32_t *dst)
 779 {
 780         WARN_ON(simd != 0);
 781
 782         wave_read_regs(
 783                 adev, wave, 0, start + SQIND_WAVE_SGPRS_OFFSET, size,
 784                 dst);
 785 }
 786
 787 static void gfx_v11_0_read_wave_vgprs(struct amdgpu_device *adev, uint32_t simd,
 788                                       uint32_t wave, uint32_t thread,
 789                                       uint32_t start, uint32_t size,
 790                                       uint32_t *dst)
 791 {
 792         wave_read_regs(
 793                 adev, wave, thread,
 794                 start + SQIND_WAVE_VGPRS_OFFSET, size, dst);
 795 }
 796
 797 static void gfx_v11_0_select_me_pipe_q(struct amdgpu_device *adev,
 798                                                                           u32 me, u32 pipe, u32 q, u32 vm)
 799 {
 800         soc21_grbm_select(adev, me, pipe, q, vm);
 801 }
 802
 803 static const struct amdgpu_gfx_funcs gfx_v11_0_gfx_funcs = {
 804         .get_gpu_clock_counter = &gfx_v11_0_get_gpu_clock_counter,
 805         .select_se_sh = &gfx_v11_0_select_se_sh,
 806         .read_wave_data = &gfx_v11_0_read_wave_data,
 807         .read_wave_sgprs = &gfx_v11_0_read_wave_sgprs,
 808         .read_wave_vgprs = &gfx_v11_0_read_wave_vgprs,
 809         .select_me_pipe_q = &gfx_v11_0_select_me_pipe_q,
 810         .update_perfmon_mgcg = &gfx_v11_0_update_perf_clk,
 811 };
 812
 813 static int gfx_v11_0_gpu_early_init(struct amdgpu_device *adev)
 814 {
 815
 816         switch (adev->ip_versions[GC_HWIP][0]) {
 817         case IP_VERSION(11, 0, 0):
 818         case IP_VERSION(11, 0, 2):
 819                 adev->gfx.config.max_hw_contexts = 8;
 820                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
 821                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
 822                 adev->gfx.config.sc_hiz_tile_fifo_size = 0;
 823                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
 824                 break;
 825         case IP_VERSION(11, 0, 3):
 826                 adev->gfx.ras = &gfx_v11_0_3_ras;
 827                 adev->gfx.config.max_hw_contexts = 8;
 828                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
 829                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
 830                 adev->gfx.config.sc_hiz_tile_fifo_size = 0;
 831                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
 832                 break;
 833         case IP_VERSION(11, 0, 1):
 834         case IP_VERSION(11, 0, 4):
 835                 adev->gfx.config.max_hw_contexts = 8;
 836                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
 837                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
 838                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x80;
 839                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x300;
 840                 break;
 841         default:
 842                 BUG();
 843                 break;
 844         }
 845
 846         return 0;
 847 }
 848
 849 static int gfx_v11_0_gfx_ring_init(struct amdgpu_device *adev, int ring_id,
 850                                    int me, int pipe, int queue)
 851 {
 852         int r;
 853         struct amdgpu_ring *ring;
 854         unsigned int irq_type;
 855
 856         ring = &adev->gfx.gfx_ring[ring_id];
 857
 858         ring->me = me;
 859         ring->pipe = pipe;
 860         ring->queue = queue;
 861
 862         ring->ring_obj = NULL;
 863         ring->use_doorbell = true;
 864
 865         if (!ring_id)
 866                 ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1;
 867         else
 868                 ring->doorbell_index = adev->doorbell_index.gfx_ring1 << 1;
 869         sprintf(ring->name, "gfx_%d.%d.%d", ring->me, ring->pipe, ring->queue);
 870
 871         irq_type = AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP + ring->pipe;
 872         r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type,
 873                              AMDGPU_RING_PRIO_DEFAULT, NULL);
 874         if (r)
 875                 return r;
 876         return 0;
 877 }
 878
 879 static int gfx_v11_0_compute_ring_init(struct amdgpu_device *adev, int ring_id,
 880                                        int mec, int pipe, int queue)
 881 {
 882         int r;
 883         unsigned irq_type;
 884         struct amdgpu_ring *ring;
 885         unsigned int hw_prio;
 886
 887         ring = &adev->gfx.compute_ring[ring_id];
 888
 889         /* mec0 is me1 */
 890         ring->me = mec + 1;
 891         ring->pipe = pipe;
 892         ring->queue = queue;
 893
 894         ring->ring_obj = NULL;
 895         ring->use_doorbell = true;
 896         ring->doorbell_index = (adev->doorbell_index.mec_ring0 + ring_id) << 1;
 897         ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr
 898                                 + (ring_id * GFX11_MEC_HPD_SIZE);
 899         sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue);
 900
 901         irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
 902                 + ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec)
 903                 + ring->pipe;
 904         hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring) ?
 905                         AMDGPU_GFX_PIPE_PRIO_HIGH : AMDGPU_GFX_PIPE_PRIO_NORMAL;
 906         /* type-2 packets are deprecated on MEC, use type-3 instead */
 907         r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type,
 908                              hw_prio, NULL);
 909         if (r)
 910                 return r;
 911
 912         return 0;
 913 }
 914
 915 static struct {
 916         SOC21_FIRMWARE_ID       id;
 917         unsigned int            offset;
 918         unsigned int            size;
 919 } rlc_autoload_info[SOC21_FIRMWARE_ID_MAX];
 920
 921 static void gfx_v11_0_parse_rlc_toc(struct amdgpu_device *adev, void *rlc_toc)
 922 {
 923         RLC_TABLE_OF_CONTENT *ucode = rlc_toc;
 924
 925         while (ucode && (ucode->id > SOC21_FIRMWARE_ID_INVALID) &&
 926                         (ucode->id < SOC21_FIRMWARE_ID_MAX)) {
 927                 rlc_autoload_info[ucode->id].id = ucode->id;
 928                 rlc_autoload_info[ucode->id].offset = ucode->offset * 4;
 929                 rlc_autoload_info[ucode->id].size = ucode->size * 4;
 930
 931                 ucode++;
 932         }
 933 }
 934
 935 static uint32_t gfx_v11_0_calc_toc_total_size(struct amdgpu_device *adev)
 936 {
 937         uint32_t total_size = 0;
 938         SOC21_FIRMWARE_ID id;
 939
 940         gfx_v11_0_parse_rlc_toc(adev, adev->psp.toc.start_addr);
 941
 942         for (id = SOC21_FIRMWARE_ID_RLC_G_UCODE; id < SOC21_FIRMWARE_ID_MAX; id++)
 943                 total_size += rlc_autoload_info[id].size;
 944
 945         /* In case the offset in rlc toc ucode is aligned */
 946         if (total_size < rlc_autoload_info[SOC21_FIRMWARE_ID_MAX-1].offset)
 947                 total_size = rlc_autoload_info[SOC21_FIRMWARE_ID_MAX-1].offset +
 948                         rlc_autoload_info[SOC21_FIRMWARE_ID_MAX-1].size;
 949
 950         return total_size;
 951 }
 952
 953 static int gfx_v11_0_rlc_autoload_buffer_init(struct amdgpu_device *adev)
 954 {
 955         int r;
 956         uint32_t total_size;
 957
 958         total_size = gfx_v11_0_calc_toc_total_size(adev);
 959
 960         r = amdgpu_bo_create_reserved(adev, total_size, 64 * 1024,
 961                                       AMDGPU_GEM_DOMAIN_VRAM |
 962                                       AMDGPU_GEM_DOMAIN_GTT,
 963                                       &adev->gfx.rlc.rlc_autoload_bo,
 964                                       &adev->gfx.rlc.rlc_autoload_gpu_addr,
 965                                       (void **)&adev->gfx.rlc.rlc_autoload_ptr);
 966
 967         if (r) {
 968                 dev_err(adev->dev, "(%d) failed to create fw autoload bo\n", r);
 969                 return r;
 970         }
 971
 972         return 0;
 973 }
 974
 975 static void gfx_v11_0_rlc_backdoor_autoload_copy_ucode(struct amdgpu_device *adev,
 976                                               SOC21_FIRMWARE_ID id,
 977                                               const void *fw_data,
 978                                               uint32_t fw_size,
 979                                               uint32_t *fw_autoload_mask)
 980 {
 981         uint32_t toc_offset;
 982         uint32_t toc_fw_size;
 983         char *ptr = adev->gfx.rlc.rlc_autoload_ptr;
 984
 985         if (id <= SOC21_FIRMWARE_ID_INVALID || id >= SOC21_FIRMWARE_ID_MAX)
 986                 return;
 987
 988         toc_offset = rlc_autoload_info[id].offset;
 989         toc_fw_size = rlc_autoload_info[id].size;
 990
 991         if (fw_size == 0)
 992                 fw_size = toc_fw_size;
 993
 994         if (fw_size > toc_fw_size)
 995                 fw_size = toc_fw_size;
 996
 997         memcpy(ptr + toc_offset, fw_data, fw_size);
 998
 999         if (fw_size < toc_fw_size)
1000                 memset(ptr + toc_offset + fw_size, 0, toc_fw_size - fw_size);
1001
1002         if ((id != SOC21_FIRMWARE_ID_RS64_PFP) && (id != SOC21_FIRMWARE_ID_RS64_ME))
1003                 *(uint64_t *)fw_autoload_mask |= 1ULL << id;
1004 }
1005
1006 static void gfx_v11_0_rlc_backdoor_autoload_copy_toc_ucode(struct amdgpu_device *adev,
1007                                                         uint32_t *fw_autoload_mask)
1008 {
1009         void *data;
1010         uint32_t size;
1011         uint64_t *toc_ptr;
1012
1013         *(uint64_t *)fw_autoload_mask |= 0x1;
1014
1015         DRM_DEBUG("rlc autoload enabled fw: 0x%llx\n", *(uint64_t *)fw_autoload_mask);
1016
1017         data = adev->psp.toc.start_addr;
1018         size = rlc_autoload_info[SOC21_FIRMWARE_ID_RLC_TOC].size;
1019
1020         toc_ptr = (uint64_t *)data + size / 8 - 1;
1021         *toc_ptr = *(uint64_t *)fw_autoload_mask;
1022
1023         gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RLC_TOC,
1024                                         data, size, fw_autoload_mask);
1025 }
1026
1027 static void gfx_v11_0_rlc_backdoor_autoload_copy_gfx_ucode(struct amdgpu_device *adev,
1028                                                         uint32_t *fw_autoload_mask)
1029 {
1030         const __le32 *fw_data;
1031         uint32_t fw_size;
1032         const struct gfx_firmware_header_v1_0 *cp_hdr;
1033         const struct gfx_firmware_header_v2_0 *cpv2_hdr;
1034         const struct rlc_firmware_header_v2_0 *rlc_hdr;
1035         const struct rlc_firmware_header_v2_2 *rlcv22_hdr;
1036         uint16_t version_major, version_minor;
1037
1038         if (adev->gfx.rs64_enable) {
1039                 /* pfp ucode */
1040                 cpv2_hdr = (const struct gfx_firmware_header_v2_0 *)
1041                         adev->gfx.pfp_fw->data;
1042                 /* instruction */
1043                 fw_data = (const __le32 *)(adev->gfx.pfp_fw->data +
1044                         le32_to_cpu(cpv2_hdr->ucode_offset_bytes));
1045                 fw_size = le32_to_cpu(cpv2_hdr->ucode_size_bytes);
1046                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_PFP,
1047                                                 fw_data, fw_size, fw_autoload_mask);
1048                 /* data */
1049                 fw_data = (const __le32 *)(adev->gfx.pfp_fw->data +
1050                         le32_to_cpu(cpv2_hdr->data_offset_bytes));
1051                 fw_size = le32_to_cpu(cpv2_hdr->data_size_bytes);
1052                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_PFP_P0_STACK,
1053                                                 fw_data, fw_size, fw_autoload_mask);
1054                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_PFP_P1_STACK,
1055                                                 fw_data, fw_size, fw_autoload_mask);
1056                 /* me ucode */
1057                 cpv2_hdr = (const struct gfx_firmware_header_v2_0 *)
1058                         adev->gfx.me_fw->data;
1059                 /* instruction */
1060                 fw_data = (const __le32 *)(adev->gfx.me_fw->data +
1061                         le32_to_cpu(cpv2_hdr->ucode_offset_bytes));
1062                 fw_size = le32_to_cpu(cpv2_hdr->ucode_size_bytes);
1063                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_ME,
1064                                                 fw_data, fw_size, fw_autoload_mask);
1065                 /* data */
1066                 fw_data = (const __le32 *)(adev->gfx.me_fw->data +
1067                         le32_to_cpu(cpv2_hdr->data_offset_bytes));
1068                 fw_size = le32_to_cpu(cpv2_hdr->data_size_bytes);
1069                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_ME_P0_STACK,
1070                                                 fw_data, fw_size, fw_autoload_mask);
1071                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_ME_P1_STACK,
1072                                                 fw_data, fw_size, fw_autoload_mask);
1073                 /* mec ucode */
1074                 cpv2_hdr = (const struct gfx_firmware_header_v2_0 *)
1075                         adev->gfx.mec_fw->data;
1076                 /* instruction */
1077                 fw_data = (const __le32 *) (adev->gfx.mec_fw->data +
1078                         le32_to_cpu(cpv2_hdr->ucode_offset_bytes));
1079                 fw_size = le32_to_cpu(cpv2_hdr->ucode_size_bytes);
1080                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_MEC,
1081                                                 fw_data, fw_size, fw_autoload_mask);
1082                 /* data */
1083                 fw_data = (const __le32 *) (adev->gfx.mec_fw->data +
1084                         le32_to_cpu(cpv2_hdr->data_offset_bytes));
1085                 fw_size = le32_to_cpu(cpv2_hdr->data_size_bytes);
1086                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_MEC_P0_STACK,
1087                                                 fw_data, fw_size, fw_autoload_mask);
1088                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_MEC_P1_STACK,
1089                                                 fw_data, fw_size, fw_autoload_mask);
1090                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_MEC_P2_STACK,
1091                                                 fw_data, fw_size, fw_autoload_mask);
1092                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_MEC_P3_STACK,
1093                                                 fw_data, fw_size, fw_autoload_mask);
1094         } else {
1095                 /* pfp ucode */
1096                 cp_hdr = (const struct gfx_firmware_header_v1_0 *)
1097                         adev->gfx.pfp_fw->data;
1098                 fw_data = (const __le32 *)(adev->gfx.pfp_fw->data +
1099                                 le32_to_cpu(cp_hdr->header.ucode_array_offset_bytes));
1100                 fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes);
1101                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_CP_PFP,
1102                                                 fw_data, fw_size, fw_autoload_mask);
1103
1104                 /* me ucode */
1105                 cp_hdr = (const struct gfx_firmware_header_v1_0 *)
1106                         adev->gfx.me_fw->data;
1107                 fw_data = (const __le32 *)(adev->gfx.me_fw->data +
1108                                 le32_to_cpu(cp_hdr->header.ucode_array_offset_bytes));
1109                 fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes);
1110                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_CP_ME,
1111                                                 fw_data, fw_size, fw_autoload_mask);
1112
1113                 /* mec ucode */
1114                 cp_hdr = (const struct gfx_firmware_header_v1_0 *)
1115                         adev->gfx.mec_fw->data;
1116                 fw_data = (const __le32 *) (adev->gfx.mec_fw->data +
1117                                 le32_to_cpu(cp_hdr->header.ucode_array_offset_bytes));
1118                 fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes) -
1119                         cp_hdr->jt_size * 4;
1120                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_CP_MEC,
1121                                                 fw_data, fw_size, fw_autoload_mask);
1122         }
1123
1124         /* rlc ucode */
1125         rlc_hdr = (const struct rlc_firmware_header_v2_0 *)
1126                 adev->gfx.rlc_fw->data;
1127         fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1128                         le32_to_cpu(rlc_hdr->header.ucode_array_offset_bytes));
1129         fw_size = le32_to_cpu(rlc_hdr->header.ucode_size_bytes);
1130         gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RLC_G_UCODE,
1131                                         fw_data, fw_size, fw_autoload_mask);
1132
1133         version_major = le16_to_cpu(rlc_hdr->header.header_version_major);
1134         version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor);
1135         if (version_major == 2) {
1136                 if (version_minor >= 2) {
1137                         rlcv22_hdr = (const struct rlc_firmware_header_v2_2 *)adev->gfx.rlc_fw->data;
1138
1139                         fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1140                                         le32_to_cpu(rlcv22_hdr->rlc_iram_ucode_offset_bytes));
1141                         fw_size = le32_to_cpu(rlcv22_hdr->rlc_iram_ucode_size_bytes);
1142                         gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RLX6_UCODE,
1143                                         fw_data, fw_size, fw_autoload_mask);
1144
1145                         fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1146                                         le32_to_cpu(rlcv22_hdr->rlc_dram_ucode_offset_bytes));
1147                         fw_size = le32_to_cpu(rlcv22_hdr->rlc_dram_ucode_size_bytes);
1148                         gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RLX6_DRAM_BOOT,
1149                                         fw_data, fw_size, fw_autoload_mask);
1150                 }
1151         }
1152 }
1153
1154 static void gfx_v11_0_rlc_backdoor_autoload_copy_sdma_ucode(struct amdgpu_device *adev,
1155                                                         uint32_t *fw_autoload_mask)
1156 {
1157         const __le32 *fw_data;
1158         uint32_t fw_size;
1159         const struct sdma_firmware_header_v2_0 *sdma_hdr;
1160
1161         sdma_hdr = (const struct sdma_firmware_header_v2_0 *)
1162                 adev->sdma.instance[0].fw->data;
1163         fw_data = (const __le32 *) (adev->sdma.instance[0].fw->data +
1164                         le32_to_cpu(sdma_hdr->header.ucode_array_offset_bytes));
1165         fw_size = le32_to_cpu(sdma_hdr->ctx_ucode_size_bytes);
1166
1167         gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev,
1168                         SOC21_FIRMWARE_ID_SDMA_UCODE_TH0, fw_data, fw_size, fw_autoload_mask);
1169
1170         fw_data = (const __le32 *) (adev->sdma.instance[0].fw->data +
1171                         le32_to_cpu(sdma_hdr->ctl_ucode_offset));
1172         fw_size = le32_to_cpu(sdma_hdr->ctl_ucode_size_bytes);
1173
1174         gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev,
1175                         SOC21_FIRMWARE_ID_SDMA_UCODE_TH1, fw_data, fw_size, fw_autoload_mask);
1176 }
1177
1178 static void gfx_v11_0_rlc_backdoor_autoload_copy_mes_ucode(struct amdgpu_device *adev,
1179                                                         uint32_t *fw_autoload_mask)
1180 {
1181         const __le32 *fw_data;
1182         unsigned fw_size;
1183         const struct mes_firmware_header_v1_0 *mes_hdr;
1184         int pipe, ucode_id, data_id;
1185
1186         for (pipe = 0; pipe < 2; pipe++) {
1187                 if (pipe==0) {
1188                         ucode_id = SOC21_FIRMWARE_ID_RS64_MES_P0;
1189                         data_id  = SOC21_FIRMWARE_ID_RS64_MES_P0_STACK;
1190                 } else {
1191                         ucode_id = SOC21_FIRMWARE_ID_RS64_MES_P1;
1192                         data_id  = SOC21_FIRMWARE_ID_RS64_MES_P1_STACK;
1193                 }
1194
1195                 mes_hdr = (const struct mes_firmware_header_v1_0 *)
1196                         adev->mes.fw[pipe]->data;
1197
1198                 fw_data = (const __le32 *)(adev->mes.fw[pipe]->data +
1199                                 le32_to_cpu(mes_hdr->mes_ucode_offset_bytes));
1200                 fw_size = le32_to_cpu(mes_hdr->mes_ucode_size_bytes);
1201
1202                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev,
1203                                 ucode_id, fw_data, fw_size, fw_autoload_mask);
1204
1205                 fw_data = (const __le32 *)(adev->mes.fw[pipe]->data +
1206                                 le32_to_cpu(mes_hdr->mes_ucode_data_offset_bytes));
1207                 fw_size = le32_to_cpu(mes_hdr->mes_ucode_data_size_bytes);
1208
1209                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev,
1210                                 data_id, fw_data, fw_size, fw_autoload_mask);
1211         }
1212 }
1213
1214 static int gfx_v11_0_rlc_backdoor_autoload_enable(struct amdgpu_device *adev)
1215 {
1216         uint32_t rlc_g_offset, rlc_g_size;
1217         uint64_t gpu_addr;
1218         uint32_t autoload_fw_id[2];
1219
1220         memset(autoload_fw_id, 0, sizeof(uint32_t) * 2);
1221
1222         /* RLC autoload sequence 2: copy ucode */
1223         gfx_v11_0_rlc_backdoor_autoload_copy_sdma_ucode(adev, autoload_fw_id);
1224         gfx_v11_0_rlc_backdoor_autoload_copy_gfx_ucode(adev, autoload_fw_id);
1225         gfx_v11_0_rlc_backdoor_autoload_copy_mes_ucode(adev, autoload_fw_id);
1226         gfx_v11_0_rlc_backdoor_autoload_copy_toc_ucode(adev, autoload_fw_id);
1227
1228         rlc_g_offset = rlc_autoload_info[SOC21_FIRMWARE_ID_RLC_G_UCODE].offset;
1229         rlc_g_size = rlc_autoload_info[SOC21_FIRMWARE_ID_RLC_G_UCODE].size;
1230         gpu_addr = adev->gfx.rlc.rlc_autoload_gpu_addr + rlc_g_offset;
1231
1232         WREG32_SOC15(GC, 0, regGFX_IMU_RLC_BOOTLOADER_ADDR_HI, upper_32_bits(gpu_addr));
1233         WREG32_SOC15(GC, 0, regGFX_IMU_RLC_BOOTLOADER_ADDR_LO, lower_32_bits(gpu_addr));
1234
1235         WREG32_SOC15(GC, 0, regGFX_IMU_RLC_BOOTLOADER_SIZE, rlc_g_size);
1236
1237         /* RLC autoload sequence 3: load IMU fw */
1238         if (adev->gfx.imu.funcs->load_microcode)
1239                 adev->gfx.imu.funcs->load_microcode(adev);
1240         /* RLC autoload sequence 4 init IMU fw */
1241         if (adev->gfx.imu.funcs->setup_imu)
1242                 adev->gfx.imu.funcs->setup_imu(adev);
1243         if (adev->gfx.imu.funcs->start_imu)
1244                 adev->gfx.imu.funcs->start_imu(adev);
1245
1246         /* RLC autoload sequence 5 disable gpa mode */
1247         gfx_v11_0_disable_gpa_mode(adev);
1248
1249         return 0;
1250 }
1251
1252 static int gfx_v11_0_sw_init(void *handle)
1253 {
1254         int i, j, k, r, ring_id = 0;
1255         struct amdgpu_kiq *kiq;
1256         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1257
1258         adev->gfxhub.funcs->init(adev);
1259
1260         switch (adev->ip_versions[GC_HWIP][0]) {
1261         case IP_VERSION(11, 0, 0):
1262         case IP_VERSION(11, 0, 2):
1263         case IP_VERSION(11, 0, 3):
1264                 adev->gfx.me.num_me = 1;
1265                 adev->gfx.me.num_pipe_per_me = 1;
1266                 adev->gfx.me.num_queue_per_pipe = 1;
1267                 adev->gfx.mec.num_mec = 2;
1268                 adev->gfx.mec.num_pipe_per_mec = 4;
1269                 adev->gfx.mec.num_queue_per_pipe = 4;
1270                 break;
1271         case IP_VERSION(11, 0, 1):
1272         case IP_VERSION(11, 0, 4):
1273                 adev->gfx.me.num_me = 1;
1274                 adev->gfx.me.num_pipe_per_me = 1;
1275                 adev->gfx.me.num_queue_per_pipe = 1;
1276                 adev->gfx.mec.num_mec = 1;
1277                 adev->gfx.mec.num_pipe_per_mec = 4;
1278                 adev->gfx.mec.num_queue_per_pipe = 4;
1279                 break;
1280         default:
1281                 adev->gfx.me.num_me = 1;
1282                 adev->gfx.me.num_pipe_per_me = 1;
1283                 adev->gfx.me.num_queue_per_pipe = 1;
1284                 adev->gfx.mec.num_mec = 1;
1285                 adev->gfx.mec.num_pipe_per_mec = 4;
1286                 adev->gfx.mec.num_queue_per_pipe = 8;
1287                 break;
1288         }
1289
1290         /* Enable CG flag in one VF mode for enabling RLC safe mode enter/exit */
1291         if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(11, 0, 3) &&
1292                 amdgpu_sriov_is_pp_one_vf(adev))
1293                 adev->cg_flags = AMD_CG_SUPPORT_GFX_CGCG;
1294
1295         /* EOP Event */
1296         r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GRBM_CP,
1297                               GFX_11_0_0__SRCID__CP_EOP_INTERRUPT,
1298                               &adev->gfx.eop_irq);
1299         if (r)
1300                 return r;
1301
1302         /* Privileged reg */
1303         r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GRBM_CP,
1304                               GFX_11_0_0__SRCID__CP_PRIV_REG_FAULT,
1305                               &adev->gfx.priv_reg_irq);
1306         if (r)
1307                 return r;
1308
1309         /* Privileged inst */
1310         r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GRBM_CP,
1311                               GFX_11_0_0__SRCID__CP_PRIV_INSTR_FAULT,
1312                               &adev->gfx.priv_inst_irq);
1313         if (r)
1314                 return r;
1315
1316         /* ECC error */
1317         r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GRBM_CP,
1318                                   GFX_11_0_0__SRCID__CP_ECC_ERROR,
1319                                   &adev->gfx.cp_ecc_error_irq);
1320         if (r)
1321                 return r;
1322
1323         /* FED error */
1324         r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GFX,
1325                                   GFX_11_0_0__SRCID__RLC_GC_FED_INTERRUPT,
1326                                   &adev->gfx.rlc_gc_fed_irq);
1327         if (r)
1328                 return r;
1329
1330         adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE;
1331
1332         if (adev->gfx.imu.funcs) {
1333                 if (adev->gfx.imu.funcs->init_microcode) {
1334                         r = adev->gfx.imu.funcs->init_microcode(adev);
1335                         if (r)
1336                                 DRM_ERROR("Failed to load imu firmware!\n");
1337                 }
1338         }
1339
1340         gfx_v11_0_me_init(adev);
1341
1342         r = gfx_v11_0_rlc_init(adev);
1343         if (r) {
1344                 DRM_ERROR("Failed to init rlc BOs!\n");
1345                 return r;
1346         }
1347
1348         r = gfx_v11_0_mec_init(adev);
1349         if (r) {
1350                 DRM_ERROR("Failed to init MEC BOs!\n");
1351                 return r;
1352         }
1353
1354         /* set up the gfx ring */
1355         for (i = 0; i < adev->gfx.me.num_me; i++) {
1356                 for (j = 0; j < adev->gfx.me.num_queue_per_pipe; j++) {
1357                         for (k = 0; k < adev->gfx.me.num_pipe_per_me; k++) {
1358                                 if (!amdgpu_gfx_is_me_queue_enabled(adev, i, k, j))
1359                                         continue;
1360
1361                                 r = gfx_v11_0_gfx_ring_init(adev, ring_id,
1362                                                             i, k, j);
1363                                 if (r)
1364                                         return r;
1365                                 ring_id++;
1366                         }
1367                 }
1368         }
1369
1370         ring_id = 0;
1371         /* set up the compute queues - allocate horizontally across pipes */
1372         for (i = 0; i < adev->gfx.mec.num_mec; ++i) {
1373                 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {
1374                         for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {
1375                                 if (!amdgpu_gfx_is_mec_queue_enabled(adev, i, k,
1376                                                                      j))
1377                                         continue;
1378
1379                                 r = gfx_v11_0_compute_ring_init(adev, ring_id,
1380                                                                 i, k, j);
1381                                 if (r)
1382                                         return r;
1383
1384                                 ring_id++;
1385                         }
1386                 }
1387         }
1388
1389         if (!adev->enable_mes_kiq) {
1390                 r = amdgpu_gfx_kiq_init(adev, GFX11_MEC_HPD_SIZE);
1391                 if (r) {
1392                         DRM_ERROR("Failed to init KIQ BOs!\n");
1393                         return r;
1394                 }
1395
1396                 kiq = &adev->gfx.kiq;
1397                 r = amdgpu_gfx_kiq_init_ring(adev, &kiq->ring, &kiq->irq);
1398                 if (r)
1399                         return r;
1400         }
1401
1402         r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct v11_compute_mqd));
1403         if (r)
1404                 return r;
1405
1406         /* allocate visible FB for rlc auto-loading fw */
1407         if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) {
1408                 r = gfx_v11_0_rlc_autoload_buffer_init(adev);
1409                 if (r)
1410                         return r;
1411         }
1412
1413         r = gfx_v11_0_gpu_early_init(adev);
1414         if (r)
1415                 return r;
1416
1417         if (amdgpu_gfx_ras_sw_init(adev)) {
1418                 dev_err(adev->dev, "Failed to initialize gfx ras block!\n");
1419                 return -EINVAL;
1420         }
1421
1422         return 0;
1423 }
1424
1425 static void gfx_v11_0_pfp_fini(struct amdgpu_device *adev)
1426 {
1427         amdgpu_bo_free_kernel(&adev->gfx.pfp.pfp_fw_obj,
1428                               &adev->gfx.pfp.pfp_fw_gpu_addr,
1429                               (void **)&adev->gfx.pfp.pfp_fw_ptr);
1430
1431         amdgpu_bo_free_kernel(&adev->gfx.pfp.pfp_fw_data_obj,
1432                               &adev->gfx.pfp.pfp_fw_data_gpu_addr,
1433                               (void **)&adev->gfx.pfp.pfp_fw_data_ptr);
1434 }
1435
1436 static void gfx_v11_0_me_fini(struct amdgpu_device *adev)
1437 {
1438         amdgpu_bo_free_kernel(&adev->gfx.me.me_fw_obj,
1439                               &adev->gfx.me.me_fw_gpu_addr,
1440                               (void **)&adev->gfx.me.me_fw_ptr);
1441
1442         amdgpu_bo_free_kernel(&adev->gfx.me.me_fw_data_obj,
1443                                &adev->gfx.me.me_fw_data_gpu_addr,
1444                                (void **)&adev->gfx.me.me_fw_data_ptr);
1445 }
1446
1447 static void gfx_v11_0_rlc_autoload_buffer_fini(struct amdgpu_device *adev)
1448 {
1449         amdgpu_bo_free_kernel(&adev->gfx.rlc.rlc_autoload_bo,
1450                         &adev->gfx.rlc.rlc_autoload_gpu_addr,
1451                         (void **)&adev->gfx.rlc.rlc_autoload_ptr);
1452 }
1453
1454 static int gfx_v11_0_sw_fini(void *handle)
1455 {
1456         int i;
1457         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1458
1459         for (i = 0; i < adev->gfx.num_gfx_rings; i++)
1460                 amdgpu_ring_fini(&adev->gfx.gfx_ring[i]);
1461         for (i = 0; i < adev->gfx.num_compute_rings; i++)
1462                 amdgpu_ring_fini(&adev->gfx.compute_ring[i]);
1463
1464         amdgpu_gfx_mqd_sw_fini(adev);
1465
1466         if (!adev->enable_mes_kiq) {
1467                 amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq.ring);
1468                 amdgpu_gfx_kiq_fini(adev);
1469         }
1470
1471         gfx_v11_0_pfp_fini(adev);
1472         gfx_v11_0_me_fini(adev);
1473         gfx_v11_0_rlc_fini(adev);
1474         gfx_v11_0_mec_fini(adev);
1475
1476         if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO)
1477                 gfx_v11_0_rlc_autoload_buffer_fini(adev);
1478
1479         gfx_v11_0_free_microcode(adev);
1480
1481         return 0;
1482 }
1483
1484 static void gfx_v11_0_select_se_sh(struct amdgpu_device *adev, u32 se_num,
1485                                    u32 sh_num, u32 instance)
1486 {
1487         u32 data;
1488
1489         if (instance == 0xffffffff)
1490                 data = REG_SET_FIELD(0, GRBM_GFX_INDEX,
1491                                      INSTANCE_BROADCAST_WRITES, 1);
1492         else
1493                 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX,
1494                                      instance);
1495
1496         if (se_num == 0xffffffff)
1497                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES,
1498                                      1);
1499         else
1500                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num);
1501
1502         if (sh_num == 0xffffffff)
1503                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SA_BROADCAST_WRITES,
1504                                      1);
1505         else
1506                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SA_INDEX, sh_num);
1507
1508         WREG32_SOC15(GC, 0, regGRBM_GFX_INDEX, data);
1509 }
1510
1511 static u32 gfx_v11_0_get_sa_active_bitmap(struct amdgpu_device *adev)
1512 {
1513         u32 gc_disabled_sa_mask, gc_user_disabled_sa_mask, sa_mask;
1514
1515         gc_disabled_sa_mask = RREG32_SOC15(GC, 0, regCC_GC_SA_UNIT_DISABLE);
1516         gc_disabled_sa_mask = REG_GET_FIELD(gc_disabled_sa_mask,
1517                                            CC_GC_SA_UNIT_DISABLE,
1518                                            SA_DISABLE);
1519         gc_user_disabled_sa_mask = RREG32_SOC15(GC, 0, regGC_USER_SA_UNIT_DISABLE);
1520         gc_user_disabled_sa_mask = REG_GET_FIELD(gc_user_disabled_sa_mask,
1521                                                  GC_USER_SA_UNIT_DISABLE,
1522                                                  SA_DISABLE);
1523         sa_mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_sh_per_se *
1524                                             adev->gfx.config.max_shader_engines);
1525
1526         return sa_mask & (~(gc_disabled_sa_mask | gc_user_disabled_sa_mask));
1527 }
1528
1529 static u32 gfx_v11_0_get_rb_active_bitmap(struct amdgpu_device *adev)
1530 {
1531         u32 gc_disabled_rb_mask, gc_user_disabled_rb_mask;
1532         u32 rb_mask;
1533
1534         gc_disabled_rb_mask = RREG32_SOC15(GC, 0, regCC_RB_BACKEND_DISABLE);
1535         gc_disabled_rb_mask = REG_GET_FIELD(gc_disabled_rb_mask,
1536                                             CC_RB_BACKEND_DISABLE,
1537                                             BACKEND_DISABLE);
1538         gc_user_disabled_rb_mask = RREG32_SOC15(GC, 0, regGC_USER_RB_BACKEND_DISABLE);
1539         gc_user_disabled_rb_mask = REG_GET_FIELD(gc_user_disabled_rb_mask,
1540                                                  GC_USER_RB_BACKEND_DISABLE,
1541                                                  BACKEND_DISABLE);
1542         rb_mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se *
1543                                             adev->gfx.config.max_shader_engines);
1544
1545         return rb_mask & (~(gc_disabled_rb_mask | gc_user_disabled_rb_mask));
1546 }
1547
1548 static void gfx_v11_0_setup_rb(struct amdgpu_device *adev)
1549 {
1550         u32 rb_bitmap_width_per_sa;
1551         u32 max_sa;
1552         u32 active_sa_bitmap;
1553         u32 global_active_rb_bitmap;
1554         u32 active_rb_bitmap = 0;
1555         u32 i;
1556
1557         /* query sa bitmap from SA_UNIT_DISABLE registers */
1558         active_sa_bitmap = gfx_v11_0_get_sa_active_bitmap(adev);
1559         /* query rb bitmap from RB_BACKEND_DISABLE registers */
1560         global_active_rb_bitmap = gfx_v11_0_get_rb_active_bitmap(adev);
1561
1562         /* generate active rb bitmap according to active sa bitmap */
1563         max_sa = adev->gfx.config.max_shader_engines *
1564                  adev->gfx.config.max_sh_per_se;
1565         rb_bitmap_width_per_sa = adev->gfx.config.max_backends_per_se /
1566                                  adev->gfx.config.max_sh_per_se;
1567         for (i = 0; i < max_sa; i++) {
1568                 if (active_sa_bitmap & (1 << i))
1569                         active_rb_bitmap |= (0x3 << (i * rb_bitmap_width_per_sa));
1570         }
1571
1572         active_rb_bitmap |= global_active_rb_bitmap;
1573         adev->gfx.config.backend_enable_mask = active_rb_bitmap;
1574         adev->gfx.config.num_rbs = hweight32(active_rb_bitmap);
1575 }
1576
1577 #define DEFAULT_SH_MEM_BASES    (0x6000)
1578 #define LDS_APP_BASE           0x1
1579 #define SCRATCH_APP_BASE       0x2
1580
1581 static void gfx_v11_0_init_compute_vmid(struct amdgpu_device *adev)
1582 {
1583         int i;
1584         uint32_t sh_mem_bases;
1585         uint32_t data;
1586
1587         /*
1588          * Configure apertures:
1589          * LDS:         0x60000000'00000000 - 0x60000001'00000000 (4GB)
1590          * Scratch:     0x60000001'00000000 - 0x60000002'00000000 (4GB)
1591          * GPUVM:       0x60010000'00000000 - 0x60020000'00000000 (1TB)
1592          */
1593         sh_mem_bases = (LDS_APP_BASE << SH_MEM_BASES__SHARED_BASE__SHIFT) |
1594                         SCRATCH_APP_BASE;
1595
1596         mutex_lock(&adev->srbm_mutex);
1597         for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
1598                 soc21_grbm_select(adev, 0, 0, 0, i);
1599                 /* CP and shaders */
1600                 WREG32_SOC15(GC, 0, regSH_MEM_CONFIG, DEFAULT_SH_MEM_CONFIG);
1601                 WREG32_SOC15(GC, 0, regSH_MEM_BASES, sh_mem_bases);
1602
1603                 /* Enable trap for each kfd vmid. */
1604                 data = RREG32_SOC15(GC, 0, regSPI_GDBG_PER_VMID_CNTL);
1605                 data = REG_SET_FIELD(data, SPI_GDBG_PER_VMID_CNTL, TRAP_EN, 1);
1606         }
1607         soc21_grbm_select(adev, 0, 0, 0, 0);
1608         mutex_unlock(&adev->srbm_mutex);
1609
1610         /* Initialize all compute VMIDs to have no GDS, GWS, or OA
1611            acccess. These should be enabled by FW for target VMIDs. */
1612         for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
1613                 WREG32_SOC15_OFFSET(GC, 0, regGDS_VMID0_BASE, 2 * i, 0);
1614                 WREG32_SOC15_OFFSET(GC, 0, regGDS_VMID0_SIZE, 2 * i, 0);
1615                 WREG32_SOC15_OFFSET(GC, 0, regGDS_GWS_VMID0, i, 0);
1616                 WREG32_SOC15_OFFSET(GC, 0, regGDS_OA_VMID0, i, 0);
1617         }
1618 }
1619
1620 static void gfx_v11_0_init_gds_vmid(struct amdgpu_device *adev)
1621 {
1622         int vmid;
1623
1624         /*
1625          * Initialize all compute and user-gfx VMIDs to have no GDS, GWS, or OA
1626          * access. Compute VMIDs should be enabled by FW for target VMIDs,
1627          * the driver can enable them for graphics. VMID0 should maintain
1628          * access so that HWS firmware can save/restore entries.
1629          */
1630         for (vmid = 1; vmid < 16; vmid++) {
1631                 WREG32_SOC15_OFFSET(GC, 0, regGDS_VMID0_BASE, 2 * vmid, 0);
1632                 WREG32_SOC15_OFFSET(GC, 0, regGDS_VMID0_SIZE, 2 * vmid, 0);
1633                 WREG32_SOC15_OFFSET(GC, 0, regGDS_GWS_VMID0, vmid, 0);
1634                 WREG32_SOC15_OFFSET(GC, 0, regGDS_OA_VMID0, vmid, 0);
1635         }
1636 }
1637
1638 static void gfx_v11_0_tcp_harvest(struct amdgpu_device *adev)
1639 {
1640         /* TODO: harvest feature to be added later. */
1641 }
1642
1643 static void gfx_v11_0_get_tcc_info(struct amdgpu_device *adev)
1644 {
1645         /* TCCs are global (not instanced). */
1646         uint32_t tcc_disable = RREG32_SOC15(GC, 0, regCGTS_TCC_DISABLE) |
1647                                RREG32_SOC15(GC, 0, regCGTS_USER_TCC_DISABLE);
1648
1649         adev->gfx.config.tcc_disabled_mask =
1650                 REG_GET_FIELD(tcc_disable, CGTS_TCC_DISABLE, TCC_DISABLE) |
1651                 (REG_GET_FIELD(tcc_disable, CGTS_TCC_DISABLE, HI_TCC_DISABLE) << 16);
1652 }
1653
1654 static void gfx_v11_0_constants_init(struct amdgpu_device *adev)
1655 {
1656         u32 tmp;
1657         int i;
1658
1659         if (!amdgpu_sriov_vf(adev))
1660                 WREG32_FIELD15_PREREG(GC, 0, GRBM_CNTL, READ_TIMEOUT, 0xff);
1661
1662         gfx_v11_0_setup_rb(adev);
1663         gfx_v11_0_get_cu_info(adev, &adev->gfx.cu_info);
1664         gfx_v11_0_get_tcc_info(adev);
1665         adev->gfx.config.pa_sc_tile_steering_override = 0;
1666
1667         /* Set whether texture coordinate truncation is conformant. */
1668         tmp = RREG32_SOC15(GC, 0, regTA_CNTL2);
1669         adev->gfx.config.ta_cntl2_truncate_coord_mode =
1670                 REG_GET_FIELD(tmp, TA_CNTL2, TRUNCATE_COORD_MODE);
1671
1672         /* XXX SH_MEM regs */
1673         /* where to put LDS, scratch, GPUVM in FSA64 space */
1674         mutex_lock(&adev->srbm_mutex);
1675         for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB_0].num_ids; i++) {
1676                 soc21_grbm_select(adev, 0, 0, 0, i);
1677                 /* CP and shaders */
1678                 WREG32_SOC15(GC, 0, regSH_MEM_CONFIG, DEFAULT_SH_MEM_CONFIG);
1679                 if (i != 0) {
1680                         tmp = REG_SET_FIELD(0, SH_MEM_BASES, PRIVATE_BASE,
1681                                 (adev->gmc.private_aperture_start >> 48));
1682                         tmp = REG_SET_FIELD(tmp, SH_MEM_BASES, SHARED_BASE,
1683                                 (adev->gmc.shared_aperture_start >> 48));
1684                         WREG32_SOC15(GC, 0, regSH_MEM_BASES, tmp);
1685                 }
1686         }
1687         soc21_grbm_select(adev, 0, 0, 0, 0);
1688
1689         mutex_unlock(&adev->srbm_mutex);
1690
1691         gfx_v11_0_init_compute_vmid(adev);
1692         gfx_v11_0_init_gds_vmid(adev);
1693 }
1694
1695 static void gfx_v11_0_enable_gui_idle_interrupt(struct amdgpu_device *adev,
1696                                                bool enable)
1697 {
1698         u32 tmp;
1699
1700         if (amdgpu_sriov_vf(adev))
1701                 return;
1702
1703         tmp = RREG32_SOC15(GC, 0, regCP_INT_CNTL_RING0);
1704
1705         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE,
1706                             enable ? 1 : 0);
1707         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE,
1708                             enable ? 1 : 0);
1709         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE,
1710                             enable ? 1 : 0);
1711         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE,
1712                             enable ? 1 : 0);
1713
1714         WREG32_SOC15(GC, 0, regCP_INT_CNTL_RING0, tmp);
1715 }
1716
1717 static int gfx_v11_0_init_csb(struct amdgpu_device *adev)
1718 {
1719         adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr);
1720
1721         WREG32_SOC15(GC, 0, regRLC_CSIB_ADDR_HI,
1722                         adev->gfx.rlc.clear_state_gpu_addr >> 32);
1723         WREG32_SOC15(GC, 0, regRLC_CSIB_ADDR_LO,
1724                         adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc);
1725         WREG32_SOC15(GC, 0, regRLC_CSIB_LENGTH, adev->gfx.rlc.clear_state_size);
1726
1727         return 0;
1728 }
1729
1730 static void gfx_v11_0_rlc_stop(struct amdgpu_device *adev)
1731 {
1732         u32 tmp = RREG32_SOC15(GC, 0, regRLC_CNTL);
1733
1734         tmp = REG_SET_FIELD(tmp, RLC_CNTL, RLC_ENABLE_F32, 0);
1735         WREG32_SOC15(GC, 0, regRLC_CNTL, tmp);
1736 }
1737
1738 static void gfx_v11_0_rlc_reset(struct amdgpu_device *adev)
1739 {
1740         WREG32_FIELD15_PREREG(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
1741         udelay(50);
1742         WREG32_FIELD15_PREREG(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 0);
1743         udelay(50);
1744 }
1745
1746 static void gfx_v11_0_rlc_smu_handshake_cntl(struct amdgpu_device *adev,
1747                                              bool enable)
1748 {
1749         uint32_t rlc_pg_cntl;
1750
1751         rlc_pg_cntl = RREG32_SOC15(GC, 0, regRLC_PG_CNTL);
1752
1753         if (!enable) {
1754                 /* RLC_PG_CNTL[23] = 0 (default)
1755                  * RLC will wait for handshake acks with SMU
1756                  * GFXOFF will be enabled
1757                  * RLC_PG_CNTL[23] = 1
1758                  * RLC will not issue any message to SMU
1759                  * hence no handshake between SMU & RLC
1760                  * GFXOFF will be disabled
1761                  */
1762                 rlc_pg_cntl |= RLC_PG_CNTL__SMU_HANDSHAKE_DISABLE_MASK;
1763         } else
1764                 rlc_pg_cntl &= ~RLC_PG_CNTL__SMU_HANDSHAKE_DISABLE_MASK;
1765         WREG32_SOC15(GC, 0, regRLC_PG_CNTL, rlc_pg_cntl);
1766 }
1767
1768 static void gfx_v11_0_rlc_start(struct amdgpu_device *adev)
1769 {
1770         /* TODO: enable rlc & smu handshake until smu
1771          * and gfxoff feature works as expected */
1772         if (!(amdgpu_pp_feature_mask & PP_GFXOFF_MASK))
1773                 gfx_v11_0_rlc_smu_handshake_cntl(adev, false);
1774
1775         WREG32_FIELD15_PREREG(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 1);
1776         udelay(50);
1777 }
1778
1779 static void gfx_v11_0_rlc_enable_srm(struct amdgpu_device *adev)
1780 {
1781         uint32_t tmp;
1782
1783         /* enable Save Restore Machine */
1784         tmp = RREG32(SOC15_REG_OFFSET(GC, 0, regRLC_SRM_CNTL));
1785         tmp |= RLC_SRM_CNTL__AUTO_INCR_ADDR_MASK;
1786         tmp |= RLC_SRM_CNTL__SRM_ENABLE_MASK;
1787         WREG32(SOC15_REG_OFFSET(GC, 0, regRLC_SRM_CNTL), tmp);
1788 }
1789
1790 static void gfx_v11_0_load_rlcg_microcode(struct amdgpu_device *adev)
1791 {
1792         const struct rlc_firmware_header_v2_0 *hdr;
1793         const __le32 *fw_data;
1794         unsigned i, fw_size;
1795
1796         hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
1797         fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1798                            le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1799         fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
1800
1801         WREG32_SOC15(GC, 0, regRLC_GPM_UCODE_ADDR,
1802                      RLCG_UCODE_LOADING_START_ADDRESS);
1803
1804         for (i = 0; i < fw_size; i++)
1805                 WREG32_SOC15(GC, 0, regRLC_GPM_UCODE_DATA,
1806                              le32_to_cpup(fw_data++));
1807
1808         WREG32_SOC15(GC, 0, regRLC_GPM_UCODE_ADDR, adev->gfx.rlc_fw_version);
1809 }
1810
1811 static void gfx_v11_0_load_rlc_iram_dram_microcode(struct amdgpu_device *adev)
1812 {
1813         const struct rlc_firmware_header_v2_2 *hdr;
1814         const __le32 *fw_data;
1815         unsigned i, fw_size;
1816         u32 tmp;
1817
1818         hdr = (const struct rlc_firmware_header_v2_2 *)adev->gfx.rlc_fw->data;
1819
1820         fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1821                         le32_to_cpu(hdr->rlc_iram_ucode_offset_bytes));
1822         fw_size = le32_to_cpu(hdr->rlc_iram_ucode_size_bytes) / 4;
1823
1824         WREG32_SOC15(GC, 0, regRLC_LX6_IRAM_ADDR, 0);
1825
1826         for (i = 0; i < fw_size; i++) {
1827                 if ((amdgpu_emu_mode == 1) && (i % 100 == 99))
1828                         msleep(1);
1829                 WREG32_SOC15(GC, 0, regRLC_LX6_IRAM_DATA,
1830                                 le32_to_cpup(fw_data++));
1831         }
1832
1833         WREG32_SOC15(GC, 0, regRLC_LX6_IRAM_ADDR, adev->gfx.rlc_fw_version);
1834
1835         fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1836                         le32_to_cpu(hdr->rlc_dram_ucode_offset_bytes));
1837         fw_size = le32_to_cpu(hdr->rlc_dram_ucode_size_bytes) / 4;
1838
1839         WREG32_SOC15(GC, 0, regRLC_LX6_DRAM_ADDR, 0);
1840         for (i = 0; i < fw_size; i++) {
1841                 if ((amdgpu_emu_mode == 1) && (i % 100 == 99))
1842                         msleep(1);
1843                 WREG32_SOC15(GC, 0, regRLC_LX6_DRAM_DATA,
1844                                 le32_to_cpup(fw_data++));
1845         }
1846
1847         WREG32_SOC15(GC, 0, regRLC_LX6_IRAM_ADDR, adev->gfx.rlc_fw_version);
1848
1849         tmp = RREG32_SOC15(GC, 0, regRLC_LX6_CNTL);
1850         tmp = REG_SET_FIELD(tmp, RLC_LX6_CNTL, PDEBUG_ENABLE, 1);
1851         tmp = REG_SET_FIELD(tmp, RLC_LX6_CNTL, BRESET, 0);
1852         WREG32_SOC15(GC, 0, regRLC_LX6_CNTL, tmp);
1853 }
1854
1855 static void gfx_v11_0_load_rlcp_rlcv_microcode(struct amdgpu_device *adev)
1856 {
1857         const struct rlc_firmware_header_v2_3 *hdr;
1858         const __le32 *fw_data;
1859         unsigned i, fw_size;
1860         u32 tmp;
1861
1862         hdr = (const struct rlc_firmware_header_v2_3 *)adev->gfx.rlc_fw->data;
1863
1864         fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1865                         le32_to_cpu(hdr->rlcp_ucode_offset_bytes));
1866         fw_size = le32_to_cpu(hdr->rlcp_ucode_size_bytes) / 4;
1867
1868         WREG32_SOC15(GC, 0, regRLC_PACE_UCODE_ADDR, 0);
1869
1870         for (i = 0; i < fw_size; i++) {
1871                 if ((amdgpu_emu_mode == 1) && (i % 100 == 99))
1872                         msleep(1);
1873                 WREG32_SOC15(GC, 0, regRLC_PACE_UCODE_DATA,
1874                                 le32_to_cpup(fw_data++));
1875         }
1876
1877         WREG32_SOC15(GC, 0, regRLC_PACE_UCODE_ADDR, adev->gfx.rlc_fw_version);
1878
1879         tmp = RREG32_SOC15(GC, 0, regRLC_GPM_THREAD_ENABLE);
1880         tmp = REG_SET_FIELD(tmp, RLC_GPM_THREAD_ENABLE, THREAD1_ENABLE, 1);
1881         WREG32_SOC15(GC, 0, regRLC_GPM_THREAD_ENABLE, tmp);
1882
1883         fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1884                         le32_to_cpu(hdr->rlcv_ucode_offset_bytes));
1885         fw_size = le32_to_cpu(hdr->rlcv_ucode_size_bytes) / 4;
1886
1887         WREG32_SOC15(GC, 0, regRLC_GPU_IOV_UCODE_ADDR, 0);
1888
1889         for (i = 0; i < fw_size; i++) {
1890                 if ((amdgpu_emu_mode == 1) && (i % 100 == 99))
1891                         msleep(1);
1892                 WREG32_SOC15(GC, 0, regRLC_GPU_IOV_UCODE_DATA,
1893                                 le32_to_cpup(fw_data++));
1894         }
1895
1896         WREG32_SOC15(GC, 0, regRLC_GPU_IOV_UCODE_ADDR, adev->gfx.rlc_fw_version);
1897
1898         tmp = RREG32_SOC15(GC, 0, regRLC_GPU_IOV_F32_CNTL);
1899         tmp = REG_SET_FIELD(tmp, RLC_GPU_IOV_F32_CNTL, ENABLE, 1);
1900         WREG32_SOC15(GC, 0, regRLC_GPU_IOV_F32_CNTL, tmp);
1901 }
1902
1903 static int gfx_v11_0_rlc_load_microcode(struct amdgpu_device *adev)
1904 {
1905         const struct rlc_firmware_header_v2_0 *hdr;
1906         uint16_t version_major;
1907         uint16_t version_minor;
1908
1909         if (!adev->gfx.rlc_fw)
1910                 return -EINVAL;
1911
1912         hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
1913         amdgpu_ucode_print_rlc_hdr(&hdr->header);
1914
1915         version_major = le16_to_cpu(hdr->header.header_version_major);
1916         version_minor = le16_to_cpu(hdr->header.header_version_minor);
1917
1918         if (version_major == 2) {
1919                 gfx_v11_0_load_rlcg_microcode(adev);
1920                 if (amdgpu_dpm == 1) {
1921                         if (version_minor >= 2)
1922                                 gfx_v11_0_load_rlc_iram_dram_microcode(adev);
1923                         if (version_minor == 3)
1924                                 gfx_v11_0_load_rlcp_rlcv_microcode(adev);
1925                 }
1926
1927                 return 0;
1928         }
1929
1930         return -EINVAL;
1931 }
1932
1933 static int gfx_v11_0_rlc_resume(struct amdgpu_device *adev)
1934 {
1935         int r;
1936
1937         if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
1938                 gfx_v11_0_init_csb(adev);
1939
1940                 if (!amdgpu_sriov_vf(adev)) /* enable RLC SRM */
1941                         gfx_v11_0_rlc_enable_srm(adev);
1942         } else {
1943                 if (amdgpu_sriov_vf(adev)) {
1944                         gfx_v11_0_init_csb(adev);
1945                         return 0;
1946                 }
1947
1948                 adev->gfx.rlc.funcs->stop(adev);
1949
1950                 /* disable CG */
1951                 WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL, 0);
1952
1953                 /* disable PG */
1954                 WREG32_SOC15(GC, 0, regRLC_PG_CNTL, 0);
1955
1956                 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
1957                         /* legacy rlc firmware loading */
1958                         r = gfx_v11_0_rlc_load_microcode(adev);
1959                         if (r)
1960                                 return r;
1961                 }
1962
1963                 gfx_v11_0_init_csb(adev);
1964
1965                 adev->gfx.rlc.funcs->start(adev);
1966         }
1967         return 0;
1968 }
1969
1970 static int gfx_v11_0_config_me_cache(struct amdgpu_device *adev, uint64_t addr)
1971 {
1972         uint32_t usec_timeout = 50000;  /* wait for 50ms */
1973         uint32_t tmp;
1974         int i;
1975
1976         /* Trigger an invalidation of the L1 instruction caches */
1977         tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
1978         tmp = REG_SET_FIELD(tmp, CP_ME_IC_OP_CNTL, INVALIDATE_CACHE, 1);
1979         WREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL, tmp);
1980
1981         /* Wait for invalidation complete */
1982         for (i = 0; i < usec_timeout; i++) {
1983                 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
1984                 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL,
1985                                         INVALIDATE_CACHE_COMPLETE))
1986                         break;
1987                 udelay(1);
1988         }
1989
1990         if (i >= usec_timeout) {
1991                 dev_err(adev->dev, "failed to invalidate instruction cache\n");
1992                 return -EINVAL;
1993         }
1994
1995         if (amdgpu_emu_mode == 1)
1996                 adev->hdp.funcs->flush_hdp(adev, NULL);
1997
1998         tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL);
1999         tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, VMID, 0);
2000         tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, CACHE_POLICY, 0);
2001         tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, EXE_DISABLE, 0);
2002         tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, ADDRESS_CLAMP, 1);
2003         WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL, tmp);
2004
2005         /* Program me ucode address into intruction cache address register */
2006         WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_LO,
2007                         lower_32_bits(addr) & 0xFFFFF000);
2008         WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_HI,
2009                         upper_32_bits(addr));
2010
2011         return 0;
2012 }
2013
2014 static int gfx_v11_0_config_pfp_cache(struct amdgpu_device *adev, uint64_t addr)
2015 {
2016         uint32_t usec_timeout = 50000;  /* wait for 50ms */
2017         uint32_t tmp;
2018         int i;
2019
2020         /* Trigger an invalidation of the L1 instruction caches */
2021         tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
2022         tmp = REG_SET_FIELD(tmp, CP_PFP_IC_OP_CNTL, INVALIDATE_CACHE, 1);
2023         WREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL, tmp);
2024
2025         /* Wait for invalidation complete */
2026         for (i = 0; i < usec_timeout; i++) {
2027                 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
2028                 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL,
2029                                         INVALIDATE_CACHE_COMPLETE))
2030                         break;
2031                 udelay(1);
2032         }
2033
2034         if (i >= usec_timeout) {
2035                 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2036                 return -EINVAL;
2037         }
2038
2039         if (amdgpu_emu_mode == 1)
2040                 adev->hdp.funcs->flush_hdp(adev, NULL);
2041
2042         tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL);
2043         tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, VMID, 0);
2044         tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, CACHE_POLICY, 0);
2045         tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, EXE_DISABLE, 0);
2046         tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, ADDRESS_CLAMP, 1);
2047         WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL, tmp);
2048
2049         /* Program pfp ucode address into intruction cache address register */
2050         WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_LO,
2051                         lower_32_bits(addr) & 0xFFFFF000);
2052         WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_HI,
2053                         upper_32_bits(addr));
2054
2055         return 0;
2056 }
2057
2058 static int gfx_v11_0_config_mec_cache(struct amdgpu_device *adev, uint64_t addr)
2059 {
2060         uint32_t usec_timeout = 50000;  /* wait for 50ms */
2061         uint32_t tmp;
2062         int i;
2063
2064         /* Trigger an invalidation of the L1 instruction caches */
2065         tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL);
2066         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_OP_CNTL, INVALIDATE_CACHE, 1);
2067
2068         WREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL, tmp);
2069
2070         /* Wait for invalidation complete */
2071         for (i = 0; i < usec_timeout; i++) {
2072                 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL);
2073                 if (1 == REG_GET_FIELD(tmp, CP_CPC_IC_OP_CNTL,
2074                                         INVALIDATE_CACHE_COMPLETE))
2075                         break;
2076                 udelay(1);
2077         }
2078
2079         if (i >= usec_timeout) {
2080                 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2081                 return -EINVAL;
2082         }
2083
2084         if (amdgpu_emu_mode == 1)
2085                 adev->hdp.funcs->flush_hdp(adev, NULL);
2086
2087         tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL);
2088         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0);
2089         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, EXE_DISABLE, 0);
2090         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, ADDRESS_CLAMP, 1);
2091         WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL, tmp);
2092
2093         /* Program mec1 ucode address into intruction cache address register */
2094         WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_LO,
2095                         lower_32_bits(addr) & 0xFFFFF000);
2096         WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_HI,
2097                         upper_32_bits(addr));
2098
2099         return 0;
2100 }
2101
2102 static int gfx_v11_0_config_pfp_cache_rs64(struct amdgpu_device *adev, uint64_t addr, uint64_t addr2)
2103 {
2104         uint32_t usec_timeout = 50000;  /* wait for 50ms */
2105         uint32_t tmp;
2106         unsigned i, pipe_id;
2107         const struct gfx_firmware_header_v2_0 *pfp_hdr;
2108
2109         pfp_hdr = (const struct gfx_firmware_header_v2_0 *)
2110                 adev->gfx.pfp_fw->data;
2111
2112         WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_LO,
2113                 lower_32_bits(addr));
2114         WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_HI,
2115                 upper_32_bits(addr));
2116
2117         tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL);
2118         tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, VMID, 0);
2119         tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, CACHE_POLICY, 0);
2120         tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, EXE_DISABLE, 0);
2121         WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL, tmp);
2122
2123         /*
2124          * Programming any of the CP_PFP_IC_BASE registers
2125          * forces invalidation of the ME L1 I$. Wait for the
2126          * invalidation complete
2127          */
2128         for (i = 0; i < usec_timeout; i++) {
2129                 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
2130                 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL,
2131                         INVALIDATE_CACHE_COMPLETE))
2132                         break;
2133                 udelay(1);
2134         }
2135
2136         if (i >= usec_timeout) {
2137                 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2138                 return -EINVAL;
2139         }
2140
2141         /* Prime the L1 instruction caches */
2142         tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
2143         tmp = REG_SET_FIELD(tmp, CP_PFP_IC_OP_CNTL, PRIME_ICACHE, 1);
2144         WREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL, tmp);
2145         /* Waiting for cache primed*/
2146         for (i = 0; i < usec_timeout; i++) {
2147                 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
2148                 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL,
2149                         ICACHE_PRIMED))
2150                         break;
2151                 udelay(1);
2152         }
2153
2154         if (i >= usec_timeout) {
2155                 dev_err(adev->dev, "failed to prime instruction cache\n");
2156                 return -EINVAL;
2157         }
2158
2159         mutex_lock(&adev->srbm_mutex);
2160         for (pipe_id = 0; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) {
2161                 soc21_grbm_select(adev, 0, pipe_id, 0, 0);
2162                 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START,
2163                         (pfp_hdr->ucode_start_addr_hi << 30) |
2164                         (pfp_hdr->ucode_start_addr_lo >> 2));
2165                 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START_HI,
2166                         pfp_hdr->ucode_start_addr_hi >> 2);
2167
2168                 /*
2169                  * Program CP_ME_CNTL to reset given PIPE to take
2170                  * effect of CP_PFP_PRGRM_CNTR_START.
2171                  */
2172                 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL);
2173                 if (pipe_id == 0)
2174                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2175                                         PFP_PIPE0_RESET, 1);
2176                 else
2177                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2178                                         PFP_PIPE1_RESET, 1);
2179                 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2180
2181                 /* Clear pfp pipe0 reset bit. */
2182                 if (pipe_id == 0)
2183                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2184                                         PFP_PIPE0_RESET, 0);
2185                 else
2186                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2187                                         PFP_PIPE1_RESET, 0);
2188                 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2189
2190                 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE0_LO,
2191                         lower_32_bits(addr2));
2192                 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE0_HI,
2193                         upper_32_bits(addr2));
2194         }
2195         soc21_grbm_select(adev, 0, 0, 0, 0);
2196         mutex_unlock(&adev->srbm_mutex);
2197
2198         tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL);
2199         tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, VMID, 0);
2200         tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, CACHE_POLICY, 0);
2201         WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL, tmp);
2202
2203         /* Invalidate the data caches */
2204         tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
2205         tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, INVALIDATE_DCACHE, 1);
2206         WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL, tmp);
2207
2208         for (i = 0; i < usec_timeout; i++) {
2209                 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
2210                 if (1 == REG_GET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL,
2211                         INVALIDATE_DCACHE_COMPLETE))
2212                         break;
2213                 udelay(1);
2214         }
2215
2216         if (i >= usec_timeout) {
2217                 dev_err(adev->dev, "failed to invalidate RS64 data cache\n");
2218                 return -EINVAL;
2219         }
2220
2221         return 0;
2222 }
2223
2224 static int gfx_v11_0_config_me_cache_rs64(struct amdgpu_device *adev, uint64_t addr, uint64_t addr2)
2225 {
2226         uint32_t usec_timeout = 50000;  /* wait for 50ms */
2227         uint32_t tmp;
2228         unsigned i, pipe_id;
2229         const struct gfx_firmware_header_v2_0 *me_hdr;
2230
2231         me_hdr = (const struct gfx_firmware_header_v2_0 *)
2232                 adev->gfx.me_fw->data;
2233
2234         WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_LO,
2235                 lower_32_bits(addr));
2236         WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_HI,
2237                 upper_32_bits(addr));
2238
2239         tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL);
2240         tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, VMID, 0);
2241         tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, CACHE_POLICY, 0);
2242         tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, EXE_DISABLE, 0);
2243         WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL, tmp);
2244
2245         /*
2246          * Programming any of the CP_ME_IC_BASE registers
2247          * forces invalidation of the ME L1 I$. Wait for the
2248          * invalidation complete
2249          */
2250         for (i = 0; i < usec_timeout; i++) {
2251                 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
2252                 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL,
2253                         INVALIDATE_CACHE_COMPLETE))
2254                         break;
2255                 udelay(1);
2256         }
2257
2258         if (i >= usec_timeout) {
2259                 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2260                 return -EINVAL;
2261         }
2262
2263         /* Prime the instruction caches */
2264         tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
2265         tmp = REG_SET_FIELD(tmp, CP_ME_IC_OP_CNTL, PRIME_ICACHE, 1);
2266         WREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL, tmp);
2267
2268         /* Waiting for instruction cache primed*/
2269         for (i = 0; i < usec_timeout; i++) {
2270                 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
2271                 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL,
2272                         ICACHE_PRIMED))
2273                         break;
2274                 udelay(1);
2275         }
2276
2277         if (i >= usec_timeout) {
2278                 dev_err(adev->dev, "failed to prime instruction cache\n");
2279                 return -EINVAL;
2280         }
2281
2282         mutex_lock(&adev->srbm_mutex);
2283         for (pipe_id = 0; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) {
2284                 soc21_grbm_select(adev, 0, pipe_id, 0, 0);
2285                 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START,
2286                         (me_hdr->ucode_start_addr_hi << 30) |
2287                         (me_hdr->ucode_start_addr_lo >> 2) );
2288                 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START_HI,
2289                         me_hdr->ucode_start_addr_hi>>2);
2290
2291                 /*
2292                  * Program CP_ME_CNTL to reset given PIPE to take
2293                  * effect of CP_PFP_PRGRM_CNTR_START.
2294                  */
2295                 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL);
2296                 if (pipe_id == 0)
2297                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2298                                         ME_PIPE0_RESET, 1);
2299                 else
2300                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2301                                         ME_PIPE1_RESET, 1);
2302                 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2303
2304                 /* Clear pfp pipe0 reset bit. */
2305                 if (pipe_id == 0)
2306                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2307                                         ME_PIPE0_RESET, 0);
2308                 else
2309                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2310                                         ME_PIPE1_RESET, 0);
2311                 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2312
2313                 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE1_LO,
2314                         lower_32_bits(addr2));
2315                 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE1_HI,
2316                         upper_32_bits(addr2));
2317         }
2318         soc21_grbm_select(adev, 0, 0, 0, 0);
2319         mutex_unlock(&adev->srbm_mutex);
2320
2321         tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL);
2322         tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, VMID, 0);
2323         tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, CACHE_POLICY, 0);
2324         WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL, tmp);
2325
2326         /* Invalidate the data caches */
2327         tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
2328         tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, INVALIDATE_DCACHE, 1);
2329         WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL, tmp);
2330
2331         for (i = 0; i < usec_timeout; i++) {
2332                 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
2333                 if (1 == REG_GET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL,
2334                         INVALIDATE_DCACHE_COMPLETE))
2335                         break;
2336                 udelay(1);
2337         }
2338
2339         if (i >= usec_timeout) {
2340                 dev_err(adev->dev, "failed to invalidate RS64 data cache\n");
2341                 return -EINVAL;
2342         }
2343
2344         return 0;
2345 }
2346
2347 static int gfx_v11_0_config_mec_cache_rs64(struct amdgpu_device *adev, uint64_t addr, uint64_t addr2)
2348 {
2349         uint32_t usec_timeout = 50000;  /* wait for 50ms */
2350         uint32_t tmp;
2351         unsigned i;
2352         const struct gfx_firmware_header_v2_0 *mec_hdr;
2353
2354         mec_hdr = (const struct gfx_firmware_header_v2_0 *)
2355                 adev->gfx.mec_fw->data;
2356
2357         tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL);
2358         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0);
2359         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, EXE_DISABLE, 0);
2360         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0);
2361         WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL, tmp);
2362
2363         tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_BASE_CNTL);
2364         tmp = REG_SET_FIELD(tmp, CP_MEC_DC_BASE_CNTL, VMID, 0);
2365         tmp = REG_SET_FIELD(tmp, CP_MEC_DC_BASE_CNTL, CACHE_POLICY, 0);
2366         WREG32_SOC15(GC, 0, regCP_MEC_DC_BASE_CNTL, tmp);
2367
2368         mutex_lock(&adev->srbm_mutex);
2369         for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) {
2370                 soc21_grbm_select(adev, 1, i, 0, 0);
2371
2372                 WREG32_SOC15(GC, 0, regCP_MEC_MDBASE_LO, addr2);
2373                 WREG32_SOC15(GC, 0, regCP_MEC_MDBASE_HI,
2374                      upper_32_bits(addr2));
2375
2376                 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START,
2377                                         mec_hdr->ucode_start_addr_lo >> 2 |
2378                                         mec_hdr->ucode_start_addr_hi << 30);
2379                 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START_HI,
2380                                         mec_hdr->ucode_start_addr_hi >> 2);
2381
2382                 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_LO, addr);
2383                 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_HI,
2384                      upper_32_bits(addr));
2385         }
2386         mutex_unlock(&adev->srbm_mutex);
2387         soc21_grbm_select(adev, 0, 0, 0, 0);
2388
2389         /* Trigger an invalidation of the L1 instruction caches */
2390         tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL);
2391         tmp = REG_SET_FIELD(tmp, CP_MEC_DC_OP_CNTL, INVALIDATE_DCACHE, 1);
2392         WREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL, tmp);
2393
2394         /* Wait for invalidation complete */
2395         for (i = 0; i < usec_timeout; i++) {
2396                 tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL);
2397                 if (1 == REG_GET_FIELD(tmp, CP_MEC_DC_OP_CNTL,
2398                                        INVALIDATE_DCACHE_COMPLETE))
2399                         break;
2400                 udelay(1);
2401         }
2402
2403         if (i >= usec_timeout) {
2404                 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2405                 return -EINVAL;
2406         }
2407
2408         /* Trigger an invalidation of the L1 instruction caches */
2409         tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL);
2410         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_OP_CNTL, INVALIDATE_CACHE, 1);
2411         WREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL, tmp);
2412
2413         /* Wait for invalidation complete */
2414         for (i = 0; i < usec_timeout; i++) {
2415                 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL);
2416                 if (1 == REG_GET_FIELD(tmp, CP_CPC_IC_OP_CNTL,
2417                                        INVALIDATE_CACHE_COMPLETE))
2418                         break;
2419                 udelay(1);
2420         }
2421
2422         if (i >= usec_timeout) {
2423                 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2424                 return -EINVAL;
2425         }
2426
2427         return 0;
2428 }
2429
2430 static void gfx_v11_0_config_gfx_rs64(struct amdgpu_device *adev)
2431 {
2432         const struct gfx_firmware_header_v2_0 *pfp_hdr;
2433         const struct gfx_firmware_header_v2_0 *me_hdr;
2434         const struct gfx_firmware_header_v2_0 *mec_hdr;
2435         uint32_t pipe_id, tmp;
2436
2437         mec_hdr = (const struct gfx_firmware_header_v2_0 *)
2438                 adev->gfx.mec_fw->data;
2439         me_hdr = (const struct gfx_firmware_header_v2_0 *)
2440                 adev->gfx.me_fw->data;
2441         pfp_hdr = (const struct gfx_firmware_header_v2_0 *)
2442                 adev->gfx.pfp_fw->data;
2443
2444         /* config pfp program start addr */
2445         for (pipe_id = 0; pipe_id < 2; pipe_id++) {
2446                 soc21_grbm_select(adev, 0, pipe_id, 0, 0);
2447                 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START,
2448                         (pfp_hdr->ucode_start_addr_hi << 30) |
2449                         (pfp_hdr->ucode_start_addr_lo >> 2));
2450                 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START_HI,
2451                         pfp_hdr->ucode_start_addr_hi >> 2);
2452         }
2453         soc21_grbm_select(adev, 0, 0, 0, 0);
2454
2455         /* reset pfp pipe */
2456         tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL);
2457         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE0_RESET, 1);
2458         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE1_RESET, 1);
2459         WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2460
2461         /* clear pfp pipe reset */
2462         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE0_RESET, 0);
2463         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE1_RESET, 0);
2464         WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2465
2466         /* config me program start addr */
2467         for (pipe_id = 0; pipe_id < 2; pipe_id++) {
2468                 soc21_grbm_select(adev, 0, pipe_id, 0, 0);
2469                 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START,
2470                         (me_hdr->ucode_start_addr_hi << 30) |
2471                         (me_hdr->ucode_start_addr_lo >> 2) );
2472                 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START_HI,
2473                         me_hdr->ucode_start_addr_hi>>2);
2474         }
2475         soc21_grbm_select(adev, 0, 0, 0, 0);
2476
2477         /* reset me pipe */
2478         tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL);
2479         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE0_RESET, 1);
2480         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE1_RESET, 1);
2481         WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2482
2483         /* clear me pipe reset */
2484         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE0_RESET, 0);
2485         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE1_RESET, 0);
2486         WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2487
2488         /* config mec program start addr */
2489         for (pipe_id = 0; pipe_id < 4; pipe_id++) {
2490                 soc21_grbm_select(adev, 1, pipe_id, 0, 0);
2491                 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START,
2492                                         mec_hdr->ucode_start_addr_lo >> 2 |
2493                                         mec_hdr->ucode_start_addr_hi << 30);
2494                 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START_HI,
2495                                         mec_hdr->ucode_start_addr_hi >> 2);
2496         }
2497         soc21_grbm_select(adev, 0, 0, 0, 0);
2498
2499         /* reset mec pipe */
2500         tmp = RREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL);
2501         tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE0_RESET, 1);
2502         tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE1_RESET, 1);
2503         tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE2_RESET, 1);
2504         tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE3_RESET, 1);
2505         WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, tmp);
2506
2507         /* clear mec pipe reset */
2508         tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE0_RESET, 0);
2509         tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE1_RESET, 0);
2510         tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE2_RESET, 0);
2511         tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE3_RESET, 0);
2512         WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, tmp);
2513 }
2514
2515 static int gfx_v11_0_wait_for_rlc_autoload_complete(struct amdgpu_device *adev)
2516 {
2517         uint32_t cp_status;
2518         uint32_t bootload_status;
2519         int i, r;
2520         uint64_t addr, addr2;
2521
2522         for (i = 0; i < adev->usec_timeout; i++) {
2523                 cp_status = RREG32_SOC15(GC, 0, regCP_STAT);
2524
2525                 if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(11, 0, 1) ||
2526                                 adev->ip_versions[GC_HWIP][0] == IP_VERSION(11, 0, 4))
2527                         bootload_status = RREG32_SOC15(GC, 0,
2528                                         regRLC_RLCS_BOOTLOAD_STATUS_gc_11_0_1);
2529                 else
2530                         bootload_status = RREG32_SOC15(GC, 0, regRLC_RLCS_BOOTLOAD_STATUS);
2531
2532                 if ((cp_status == 0) &&
2533                     (REG_GET_FIELD(bootload_status,
2534                         RLC_RLCS_BOOTLOAD_STATUS, BOOTLOAD_COMPLETE) == 1)) {
2535                         break;
2536                 }
2537                 udelay(1);
2538         }
2539
2540         if (i >= adev->usec_timeout) {
2541                 dev_err(adev->dev, "rlc autoload: gc ucode autoload timeout\n");
2542                 return -ETIMEDOUT;
2543         }
2544
2545         if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) {
2546                 if (adev->gfx.rs64_enable) {
2547                         addr = adev->gfx.rlc.rlc_autoload_gpu_addr +
2548                                 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_ME].offset;
2549                         addr2 = adev->gfx.rlc.rlc_autoload_gpu_addr +
2550                                 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_ME_P0_STACK].offset;
2551                         r = gfx_v11_0_config_me_cache_rs64(adev, addr, addr2);
2552                         if (r)
2553                                 return r;
2554                         addr = adev->gfx.rlc.rlc_autoload_gpu_addr +
2555                                 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_PFP].offset;
2556                         addr2 = adev->gfx.rlc.rlc_autoload_gpu_addr +
2557                                 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_PFP_P0_STACK].offset;
2558                         r = gfx_v11_0_config_pfp_cache_rs64(adev, addr, addr2);
2559                         if (r)
2560                                 return r;
2561                         addr = adev->gfx.rlc.rlc_autoload_gpu_addr +
2562                                 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_MEC].offset;
2563                         addr2 = adev->gfx.rlc.rlc_autoload_gpu_addr +
2564                                 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_MEC_P0_STACK].offset;
2565                         r = gfx_v11_0_config_mec_cache_rs64(adev, addr, addr2);
2566                         if (r)
2567                                 return r;
2568                 } else {
2569                         addr = adev->gfx.rlc.rlc_autoload_gpu_addr +
2570                                 rlc_autoload_info[SOC21_FIRMWARE_ID_CP_ME].offset;
2571                         r = gfx_v11_0_config_me_cache(adev, addr);
2572                         if (r)
2573                                 return r;
2574                         addr = adev->gfx.rlc.rlc_autoload_gpu_addr +
2575                                 rlc_autoload_info[SOC21_FIRMWARE_ID_CP_PFP].offset;
2576                         r = gfx_v11_0_config_pfp_cache(adev, addr);
2577                         if (r)
2578                                 return r;
2579                         addr = adev->gfx.rlc.rlc_autoload_gpu_addr +
2580                                 rlc_autoload_info[SOC21_FIRMWARE_ID_CP_MEC].offset;
2581                         r = gfx_v11_0_config_mec_cache(adev, addr);
2582                         if (r)
2583                                 return r;
2584                 }
2585         }
2586
2587         return 0;
2588 }
2589
2590 static int gfx_v11_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable)
2591 {
2592         int i;
2593         u32 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL);
2594
2595         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, enable ? 0 : 1);
2596         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, enable ? 0 : 1);
2597         WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2598
2599         for (i = 0; i < adev->usec_timeout; i++) {
2600                 if (RREG32_SOC15(GC, 0, regCP_STAT) == 0)
2601                         break;
2602                 udelay(1);
2603         }
2604
2605         if (i >= adev->usec_timeout)
2606                 DRM_ERROR("failed to %s cp gfx\n", enable ? "unhalt" : "halt");
2607
2608         return 0;
2609 }
2610
2611 static int gfx_v11_0_cp_gfx_load_pfp_microcode(struct amdgpu_device *adev)
2612 {
2613         int r;
2614         const struct gfx_firmware_header_v1_0 *pfp_hdr;
2615         const __le32 *fw_data;
2616         unsigned i, fw_size;
2617
2618         pfp_hdr = (const struct gfx_firmware_header_v1_0 *)
2619                 adev->gfx.pfp_fw->data;
2620
2621         amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header);
2622
2623         fw_data = (const __le32 *)(adev->gfx.pfp_fw->data +
2624                 le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
2625         fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes);
2626
2627         r = amdgpu_bo_create_reserved(adev, pfp_hdr->header.ucode_size_bytes,
2628                                       PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
2629                                       &adev->gfx.pfp.pfp_fw_obj,
2630                                       &adev->gfx.pfp.pfp_fw_gpu_addr,
2631                                       (void **)&adev->gfx.pfp.pfp_fw_ptr);
2632         if (r) {
2633                 dev_err(adev->dev, "(%d) failed to create pfp fw bo\n", r);
2634                 gfx_v11_0_pfp_fini(adev);
2635                 return r;
2636         }
2637
2638         memcpy(adev->gfx.pfp.pfp_fw_ptr, fw_data, fw_size);
2639
2640         amdgpu_bo_kunmap(adev->gfx.pfp.pfp_fw_obj);
2641         amdgpu_bo_unreserve(adev->gfx.pfp.pfp_fw_obj);
2642
2643         gfx_v11_0_config_pfp_cache(adev, adev->gfx.pfp.pfp_fw_gpu_addr);
2644
2645         WREG32_SOC15(GC, 0, regCP_HYP_PFP_UCODE_ADDR, 0);
2646
2647         for (i = 0; i < pfp_hdr->jt_size; i++)
2648                 WREG32_SOC15(GC, 0, regCP_HYP_PFP_UCODE_DATA,
2649                              le32_to_cpup(fw_data + pfp_hdr->jt_offset + i));
2650
2651         WREG32_SOC15(GC, 0, regCP_HYP_PFP_UCODE_ADDR, adev->gfx.pfp_fw_version);
2652
2653         return 0;
2654 }
2655
2656 static int gfx_v11_0_cp_gfx_load_pfp_microcode_rs64(struct amdgpu_device *adev)
2657 {
2658         int r;
2659         const struct gfx_firmware_header_v2_0 *pfp_hdr;
2660         const __le32 *fw_ucode, *fw_data;
2661         unsigned i, pipe_id, fw_ucode_size, fw_data_size;
2662         uint32_t tmp;
2663         uint32_t usec_timeout = 50000;  /* wait for 50ms */
2664
2665         pfp_hdr = (const struct gfx_firmware_header_v2_0 *)
2666                 adev->gfx.pfp_fw->data;
2667
2668         amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header);
2669
2670         /* instruction */
2671         fw_ucode = (const __le32 *)(adev->gfx.pfp_fw->data +
2672                 le32_to_cpu(pfp_hdr->ucode_offset_bytes));
2673         fw_ucode_size = le32_to_cpu(pfp_hdr->ucode_size_bytes);
2674         /* data */
2675         fw_data = (const __le32 *)(adev->gfx.pfp_fw->data +
2676                 le32_to_cpu(pfp_hdr->data_offset_bytes));
2677         fw_data_size = le32_to_cpu(pfp_hdr->data_size_bytes);
2678
2679         /* 64kb align */
2680         r = amdgpu_bo_create_reserved(adev, fw_ucode_size,
2681                                       64 * 1024,
2682                                       AMDGPU_GEM_DOMAIN_VRAM |
2683                                       AMDGPU_GEM_DOMAIN_GTT,
2684                                       &adev->gfx.pfp.pfp_fw_obj,
2685                                       &adev->gfx.pfp.pfp_fw_gpu_addr,
2686                                       (void **)&adev->gfx.pfp.pfp_fw_ptr);
2687         if (r) {
2688                 dev_err(adev->dev, "(%d) failed to create pfp ucode fw bo\n", r);
2689                 gfx_v11_0_pfp_fini(adev);
2690                 return r;
2691         }
2692
2693         r = amdgpu_bo_create_reserved(adev, fw_data_size,
2694                                       64 * 1024,
2695                                       AMDGPU_GEM_DOMAIN_VRAM |
2696                                       AMDGPU_GEM_DOMAIN_GTT,
2697                                       &adev->gfx.pfp.pfp_fw_data_obj,
2698                                       &adev->gfx.pfp.pfp_fw_data_gpu_addr,
2699                                       (void **)&adev->gfx.pfp.pfp_fw_data_ptr);
2700         if (r) {
2701                 dev_err(adev->dev, "(%d) failed to create pfp data fw bo\n", r);
2702                 gfx_v11_0_pfp_fini(adev);
2703                 return r;
2704         }
2705
2706         memcpy(adev->gfx.pfp.pfp_fw_ptr, fw_ucode, fw_ucode_size);
2707         memcpy(adev->gfx.pfp.pfp_fw_data_ptr, fw_data, fw_data_size);
2708
2709         amdgpu_bo_kunmap(adev->gfx.pfp.pfp_fw_obj);
2710         amdgpu_bo_kunmap(adev->gfx.pfp.pfp_fw_data_obj);
2711         amdgpu_bo_unreserve(adev->gfx.pfp.pfp_fw_obj);
2712         amdgpu_bo_unreserve(adev->gfx.pfp.pfp_fw_data_obj);
2713
2714         if (amdgpu_emu_mode == 1)
2715                 adev->hdp.funcs->flush_hdp(adev, NULL);
2716
2717         WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_LO,
2718                 lower_32_bits(adev->gfx.pfp.pfp_fw_gpu_addr));
2719         WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_HI,
2720                 upper_32_bits(adev->gfx.pfp.pfp_fw_gpu_addr));
2721
2722         tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL);
2723         tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, VMID, 0);
2724         tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, CACHE_POLICY, 0);
2725         tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, EXE_DISABLE, 0);
2726         WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL, tmp);
2727
2728         /*
2729          * Programming any of the CP_PFP_IC_BASE registers
2730          * forces invalidation of the ME L1 I$. Wait for the
2731          * invalidation complete
2732          */
2733         for (i = 0; i < usec_timeout; i++) {
2734                 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
2735                 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL,
2736                         INVALIDATE_CACHE_COMPLETE))
2737                         break;
2738                 udelay(1);
2739         }
2740
2741         if (i >= usec_timeout) {
2742                 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2743                 return -EINVAL;
2744         }
2745
2746         /* Prime the L1 instruction caches */
2747         tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
2748         tmp = REG_SET_FIELD(tmp, CP_PFP_IC_OP_CNTL, PRIME_ICACHE, 1);
2749         WREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL, tmp);
2750         /* Waiting for cache primed*/
2751         for (i = 0; i < usec_timeout; i++) {
2752                 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
2753                 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL,
2754                         ICACHE_PRIMED))
2755                         break;
2756                 udelay(1);
2757         }
2758
2759         if (i >= usec_timeout) {
2760                 dev_err(adev->dev, "failed to prime instruction cache\n");
2761                 return -EINVAL;
2762         }
2763
2764         mutex_lock(&adev->srbm_mutex);
2765         for (pipe_id = 0; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) {
2766                 soc21_grbm_select(adev, 0, pipe_id, 0, 0);
2767                 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START,
2768                         (pfp_hdr->ucode_start_addr_hi << 30) |
2769                         (pfp_hdr->ucode_start_addr_lo >> 2) );
2770                 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START_HI,
2771                         pfp_hdr->ucode_start_addr_hi>>2);
2772
2773                 /*
2774                  * Program CP_ME_CNTL to reset given PIPE to take
2775                  * effect of CP_PFP_PRGRM_CNTR_START.
2776                  */
2777                 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL);
2778                 if (pipe_id == 0)
2779                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2780                                         PFP_PIPE0_RESET, 1);
2781                 else
2782                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2783                                         PFP_PIPE1_RESET, 1);
2784                 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2785
2786                 /* Clear pfp pipe0 reset bit. */
2787                 if (pipe_id == 0)
2788                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2789                                         PFP_PIPE0_RESET, 0);
2790                 else
2791                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2792                                         PFP_PIPE1_RESET, 0);
2793                 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2794
2795                 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE0_LO,
2796                         lower_32_bits(adev->gfx.pfp.pfp_fw_data_gpu_addr));
2797                 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE0_HI,
2798                         upper_32_bits(adev->gfx.pfp.pfp_fw_data_gpu_addr));
2799         }
2800         soc21_grbm_select(adev, 0, 0, 0, 0);
2801         mutex_unlock(&adev->srbm_mutex);
2802
2803         tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL);
2804         tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, VMID, 0);
2805         tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, CACHE_POLICY, 0);
2806         WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL, tmp);
2807
2808         /* Invalidate the data caches */
2809         tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
2810         tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, INVALIDATE_DCACHE, 1);
2811         WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL, tmp);
2812
2813         for (i = 0; i < usec_timeout; i++) {
2814                 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
2815                 if (1 == REG_GET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL,
2816                         INVALIDATE_DCACHE_COMPLETE))
2817                         break;
2818                 udelay(1);
2819         }
2820
2821         if (i >= usec_timeout) {
2822                 dev_err(adev->dev, "failed to invalidate RS64 data cache\n");
2823                 return -EINVAL;
2824         }
2825
2826         return 0;
2827 }
2828
2829 static int gfx_v11_0_cp_gfx_load_me_microcode(struct amdgpu_device *adev)
2830 {
2831         int r;
2832         const struct gfx_firmware_header_v1_0 *me_hdr;
2833         const __le32 *fw_data;
2834         unsigned i, fw_size;
2835
2836         me_hdr = (const struct gfx_firmware_header_v1_0 *)
2837                 adev->gfx.me_fw->data;
2838
2839         amdgpu_ucode_print_gfx_hdr(&me_hdr->header);
2840
2841         fw_data = (const __le32 *)(adev->gfx.me_fw->data +
2842                 le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
2843         fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes);
2844
2845         r = amdgpu_bo_create_reserved(adev, me_hdr->header.ucode_size_bytes,
2846                                       PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
2847                                       &adev->gfx.me.me_fw_obj,
2848                                       &adev->gfx.me.me_fw_gpu_addr,
2849                                       (void **)&adev->gfx.me.me_fw_ptr);
2850         if (r) {
2851                 dev_err(adev->dev, "(%d) failed to create me fw bo\n", r);
2852                 gfx_v11_0_me_fini(adev);
2853                 return r;
2854         }
2855
2856         memcpy(adev->gfx.me.me_fw_ptr, fw_data, fw_size);
2857
2858         amdgpu_bo_kunmap(adev->gfx.me.me_fw_obj);
2859         amdgpu_bo_unreserve(adev->gfx.me.me_fw_obj);
2860
2861         gfx_v11_0_config_me_cache(adev, adev->gfx.me.me_fw_gpu_addr);
2862
2863         WREG32_SOC15(GC, 0, regCP_HYP_ME_UCODE_ADDR, 0);
2864
2865         for (i = 0; i < me_hdr->jt_size; i++)
2866                 WREG32_SOC15(GC, 0, regCP_HYP_ME_UCODE_DATA,
2867                              le32_to_cpup(fw_data + me_hdr->jt_offset + i));
2868
2869         WREG32_SOC15(GC, 0, regCP_HYP_ME_UCODE_ADDR, adev->gfx.me_fw_version);
2870
2871         return 0;
2872 }
2873
2874 static int gfx_v11_0_cp_gfx_load_me_microcode_rs64(struct amdgpu_device *adev)
2875 {
2876         int r;
2877         const struct gfx_firmware_header_v2_0 *me_hdr;
2878         const __le32 *fw_ucode, *fw_data;
2879         unsigned i, pipe_id, fw_ucode_size, fw_data_size;
2880         uint32_t tmp;
2881         uint32_t usec_timeout = 50000;  /* wait for 50ms */
2882
2883         me_hdr = (const struct gfx_firmware_header_v2_0 *)
2884                 adev->gfx.me_fw->data;
2885
2886         amdgpu_ucode_print_gfx_hdr(&me_hdr->header);
2887
2888         /* instruction */
2889         fw_ucode = (const __le32 *)(adev->gfx.me_fw->data +
2890                 le32_to_cpu(me_hdr->ucode_offset_bytes));
2891         fw_ucode_size = le32_to_cpu(me_hdr->ucode_size_bytes);
2892         /* data */
2893         fw_data = (const __le32 *)(adev->gfx.me_fw->data +
2894                 le32_to_cpu(me_hdr->data_offset_bytes));
2895         fw_data_size = le32_to_cpu(me_hdr->data_size_bytes);
2896
2897         /* 64kb align*/
2898         r = amdgpu_bo_create_reserved(adev, fw_ucode_size,
2899                                       64 * 1024,
2900                                       AMDGPU_GEM_DOMAIN_VRAM |
2901                                       AMDGPU_GEM_DOMAIN_GTT,
2902                                       &adev->gfx.me.me_fw_obj,
2903                                       &adev->gfx.me.me_fw_gpu_addr,
2904                                       (void **)&adev->gfx.me.me_fw_ptr);
2905         if (r) {
2906                 dev_err(adev->dev, "(%d) failed to create me ucode bo\n", r);
2907                 gfx_v11_0_me_fini(adev);
2908                 return r;
2909         }
2910
2911         r = amdgpu_bo_create_reserved(adev, fw_data_size,
2912                                       64 * 1024,
2913                                       AMDGPU_GEM_DOMAIN_VRAM |
2914                                       AMDGPU_GEM_DOMAIN_GTT,
2915                                       &adev->gfx.me.me_fw_data_obj,
2916                                       &adev->gfx.me.me_fw_data_gpu_addr,
2917                                       (void **)&adev->gfx.me.me_fw_data_ptr);
2918         if (r) {
2919                 dev_err(adev->dev, "(%d) failed to create me data bo\n", r);
2920                 gfx_v11_0_pfp_fini(adev);
2921                 return r;
2922         }
2923
2924         memcpy(adev->gfx.me.me_fw_ptr, fw_ucode, fw_ucode_size);
2925         memcpy(adev->gfx.me.me_fw_data_ptr, fw_data, fw_data_size);
2926
2927         amdgpu_bo_kunmap(adev->gfx.me.me_fw_obj);
2928         amdgpu_bo_kunmap(adev->gfx.me.me_fw_data_obj);
2929         amdgpu_bo_unreserve(adev->gfx.me.me_fw_obj);
2930         amdgpu_bo_unreserve(adev->gfx.me.me_fw_data_obj);
2931
2932         if (amdgpu_emu_mode == 1)
2933                 adev->hdp.funcs->flush_hdp(adev, NULL);
2934
2935         WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_LO,
2936                 lower_32_bits(adev->gfx.me.me_fw_gpu_addr));
2937         WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_HI,
2938                 upper_32_bits(adev->gfx.me.me_fw_gpu_addr));
2939
2940         tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL);
2941         tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, VMID, 0);
2942         tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, CACHE_POLICY, 0);
2943         tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, EXE_DISABLE, 0);
2944         WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL, tmp);
2945
2946         /*
2947          * Programming any of the CP_ME_IC_BASE registers
2948          * forces invalidation of the ME L1 I$. Wait for the
2949          * invalidation complete
2950          */
2951         for (i = 0; i < usec_timeout; i++) {
2952                 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
2953                 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL,
2954                         INVALIDATE_CACHE_COMPLETE))
2955                         break;
2956                 udelay(1);
2957         }
2958
2959         if (i >= usec_timeout) {
2960                 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2961                 return -EINVAL;
2962         }
2963
2964         /* Prime the instruction caches */
2965         tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
2966         tmp = REG_SET_FIELD(tmp, CP_ME_IC_OP_CNTL, PRIME_ICACHE, 1);
2967         WREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL, tmp);
2968
2969         /* Waiting for instruction cache primed*/
2970         for (i = 0; i < usec_timeout; i++) {
2971                 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
2972                 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL,
2973                         ICACHE_PRIMED))
2974                         break;
2975                 udelay(1);
2976         }
2977
2978         if (i >= usec_timeout) {
2979                 dev_err(adev->dev, "failed to prime instruction cache\n");
2980                 return -EINVAL;
2981         }
2982
2983         mutex_lock(&adev->srbm_mutex);
2984         for (pipe_id = 0; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) {
2985                 soc21_grbm_select(adev, 0, pipe_id, 0, 0);
2986                 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START,
2987                         (me_hdr->ucode_start_addr_hi << 30) |
2988                         (me_hdr->ucode_start_addr_lo >> 2) );
2989                 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START_HI,
2990                         me_hdr->ucode_start_addr_hi>>2);
2991
2992                 /*
2993                  * Program CP_ME_CNTL to reset given PIPE to take
2994                  * effect of CP_PFP_PRGRM_CNTR_START.
2995                  */
2996                 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL);
2997                 if (pipe_id == 0)
2998                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2999                                         ME_PIPE0_RESET, 1);
3000                 else
3001                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
3002                                         ME_PIPE1_RESET, 1);
3003                 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
3004
3005                 /* Clear pfp pipe0 reset bit. */
3006                 if (pipe_id == 0)
3007                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
3008                                         ME_PIPE0_RESET, 0);
3009                 else
3010                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
3011                                         ME_PIPE1_RESET, 0);
3012                 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
3013
3014                 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE1_LO,
3015                         lower_32_bits(adev->gfx.me.me_fw_data_gpu_addr));
3016                 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE1_HI,
3017                         upper_32_bits(adev->gfx.me.me_fw_data_gpu_addr));
3018         }
3019         soc21_grbm_select(adev, 0, 0, 0, 0);
3020         mutex_unlock(&adev->srbm_mutex);
3021
3022         tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL);
3023         tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, VMID, 0);
3024         tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, CACHE_POLICY, 0);
3025         WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL, tmp);
3026
3027         /* Invalidate the data caches */
3028         tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
3029         tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, INVALIDATE_DCACHE, 1);
3030         WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL, tmp);
3031
3032         for (i = 0; i < usec_timeout; i++) {
3033                 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
3034                 if (1 == REG_GET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL,
3035                         INVALIDATE_DCACHE_COMPLETE))
3036                         break;
3037                 udelay(1);
3038         }
3039
3040         if (i >= usec_timeout) {
3041                 dev_err(adev->dev, "failed to invalidate RS64 data cache\n");
3042                 return -EINVAL;
3043         }
3044
3045         return 0;
3046 }
3047
3048 static int gfx_v11_0_cp_gfx_load_microcode(struct amdgpu_device *adev)
3049 {
3050         int r;
3051
3052         if (!adev->gfx.me_fw || !adev->gfx.pfp_fw)
3053                 return -EINVAL;
3054
3055         gfx_v11_0_cp_gfx_enable(adev, false);
3056
3057         if (adev->gfx.rs64_enable)
3058                 r = gfx_v11_0_cp_gfx_load_pfp_microcode_rs64(adev);
3059         else
3060                 r = gfx_v11_0_cp_gfx_load_pfp_microcode(adev);
3061         if (r) {
3062                 dev_err(adev->dev, "(%d) failed to load pfp fw\n", r);
3063                 return r;
3064         }
3065
3066         if (adev->gfx.rs64_enable)
3067                 r = gfx_v11_0_cp_gfx_load_me_microcode_rs64(adev);
3068         else
3069                 r = gfx_v11_0_cp_gfx_load_me_microcode(adev);
3070         if (r) {
3071                 dev_err(adev->dev, "(%d) failed to load me fw\n", r);
3072                 return r;
3073         }
3074
3075         return 0;
3076 }
3077
3078 static int gfx_v11_0_cp_gfx_start(struct amdgpu_device *adev)
3079 {
3080         struct amdgpu_ring *ring;
3081         const struct cs_section_def *sect = NULL;
3082         const struct cs_extent_def *ext = NULL;
3083         int r, i;
3084         int ctx_reg_offset;
3085
3086         /* init the CP */
3087         WREG32_SOC15(GC, 0, regCP_MAX_CONTEXT,
3088                      adev->gfx.config.max_hw_contexts - 1);
3089         WREG32_SOC15(GC, 0, regCP_DEVICE_ID, 1);
3090
3091         if (!amdgpu_async_gfx_ring)
3092                 gfx_v11_0_cp_gfx_enable(adev, true);
3093
3094         ring = &adev->gfx.gfx_ring[0];
3095         r = amdgpu_ring_alloc(ring, gfx_v11_0_get_csb_size(adev));
3096         if (r) {
3097                 DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r);
3098                 return r;
3099         }
3100
3101         amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3102         amdgpu_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3103
3104         amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
3105         amdgpu_ring_write(ring, 0x80000000);
3106         amdgpu_ring_write(ring, 0x80000000);
3107
3108         for (sect = gfx11_cs_data; sect->section != NULL; ++sect) {
3109                 for (ext = sect->section; ext->extent != NULL; ++ext) {
3110                         if (sect->id == SECT_CONTEXT) {
3111                                 amdgpu_ring_write(ring,
3112                                                   PACKET3(PACKET3_SET_CONTEXT_REG,
3113                                                           ext->reg_count));
3114                                 amdgpu_ring_write(ring, ext->reg_index -
3115                                                   PACKET3_SET_CONTEXT_REG_START);
3116                                 for (i = 0; i < ext->reg_count; i++)
3117                                         amdgpu_ring_write(ring, ext->extent[i]);
3118                         }
3119                 }
3120         }
3121
3122         ctx_reg_offset =
3123                 SOC15_REG_OFFSET(GC, 0, regPA_SC_TILE_STEERING_OVERRIDE) - PACKET3_SET_CONTEXT_REG_START;
3124         amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 1));
3125         amdgpu_ring_write(ring, ctx_reg_offset);
3126         amdgpu_ring_write(ring, adev->gfx.config.pa_sc_tile_steering_override);
3127
3128         amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3129         amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3130
3131         amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3132         amdgpu_ring_write(ring, 0);
3133
3134         amdgpu_ring_commit(ring);
3135
3136         /* submit cs packet to copy state 0 to next available state */
3137         if (adev->gfx.num_gfx_rings > 1) {
3138                 /* maximum supported gfx ring is 2 */
3139                 ring = &adev->gfx.gfx_ring[1];
3140                 r = amdgpu_ring_alloc(ring, 2);
3141                 if (r) {
3142                         DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r);
3143                         return r;
3144                 }
3145
3146                 amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3147                 amdgpu_ring_write(ring, 0);
3148
3149                 amdgpu_ring_commit(ring);
3150         }
3151         return 0;
3152 }
3153
3154 static void gfx_v11_0_cp_gfx_switch_pipe(struct amdgpu_device *adev,
3155                                          CP_PIPE_ID pipe)
3156 {
3157         u32 tmp;
3158
3159         tmp = RREG32_SOC15(GC, 0, regGRBM_GFX_CNTL);
3160         tmp = REG_SET_FIELD(tmp, GRBM_GFX_CNTL, PIPEID, pipe);
3161
3162         WREG32_SOC15(GC, 0, regGRBM_GFX_CNTL, tmp);
3163 }
3164
3165 static void gfx_v11_0_cp_gfx_set_doorbell(struct amdgpu_device *adev,
3166                                           struct amdgpu_ring *ring)
3167 {
3168         u32 tmp;
3169
3170         tmp = RREG32_SOC15(GC, 0, regCP_RB_DOORBELL_CONTROL);
3171         if (ring->use_doorbell) {
3172                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3173                                     DOORBELL_OFFSET, ring->doorbell_index);
3174                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3175                                     DOORBELL_EN, 1);
3176         } else {
3177                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3178                                     DOORBELL_EN, 0);
3179         }
3180         WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_CONTROL, tmp);
3181
3182         tmp = REG_SET_FIELD(0, CP_RB_DOORBELL_RANGE_LOWER,
3183                             DOORBELL_RANGE_LOWER, ring->doorbell_index);
3184         WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_RANGE_LOWER, tmp);
3185
3186         WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_RANGE_UPPER,
3187                      CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK);
3188 }
3189
3190 static int gfx_v11_0_cp_gfx_resume(struct amdgpu_device *adev)
3191 {
3192         struct amdgpu_ring *ring;
3193         u32 tmp;
3194         u32 rb_bufsz;
3195         u64 rb_addr, rptr_addr, wptr_gpu_addr;
3196         u32 i;
3197
3198         /* Set the write pointer delay */
3199         WREG32_SOC15(GC, 0, regCP_RB_WPTR_DELAY, 0);
3200
3201         /* set the RB to use vmid 0 */
3202         WREG32_SOC15(GC, 0, regCP_RB_VMID, 0);
3203
3204         /* Init gfx ring 0 for pipe 0 */
3205         mutex_lock(&adev->srbm_mutex);
3206         gfx_v11_0_cp_gfx_switch_pipe(adev, PIPE_ID0);
3207
3208         /* Set ring buffer size */
3209         ring = &adev->gfx.gfx_ring[0];
3210         rb_bufsz = order_base_2(ring->ring_size / 8);
3211         tmp = REG_SET_FIELD(0, CP_RB0_CNTL, RB_BUFSZ, rb_bufsz);
3212         tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, RB_BLKSZ, rb_bufsz - 2);
3213         WREG32_SOC15(GC, 0, regCP_RB0_CNTL, tmp);
3214
3215         /* Initialize the ring buffer's write pointers */
3216         ring->wptr = 0;
3217         WREG32_SOC15(GC, 0, regCP_RB0_WPTR, lower_32_bits(ring->wptr));
3218         WREG32_SOC15(GC, 0, regCP_RB0_WPTR_HI, upper_32_bits(ring->wptr));
3219
3220         /* set the wb address wether it's enabled or not */
3221         rptr_addr = ring->rptr_gpu_addr;
3222         WREG32_SOC15(GC, 0, regCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr));
3223         WREG32_SOC15(GC, 0, regCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) &
3224                      CP_RB_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK);
3225
3226         wptr_gpu_addr = ring->wptr_gpu_addr;
3227         WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_ADDR_LO,
3228                      lower_32_bits(wptr_gpu_addr));
3229         WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_ADDR_HI,
3230                      upper_32_bits(wptr_gpu_addr));
3231
3232         mdelay(1);
3233         WREG32_SOC15(GC, 0, regCP_RB0_CNTL, tmp);
3234
3235         rb_addr = ring->gpu_addr >> 8;
3236         WREG32_SOC15(GC, 0, regCP_RB0_BASE, rb_addr);
3237         WREG32_SOC15(GC, 0, regCP_RB0_BASE_HI, upper_32_bits(rb_addr));
3238
3239         WREG32_SOC15(GC, 0, regCP_RB_ACTIVE, 1);
3240
3241         gfx_v11_0_cp_gfx_set_doorbell(adev, ring);
3242         mutex_unlock(&adev->srbm_mutex);
3243
3244         /* Init gfx ring 1 for pipe 1 */
3245         if (adev->gfx.num_gfx_rings > 1) {
3246                 mutex_lock(&adev->srbm_mutex);
3247                 gfx_v11_0_cp_gfx_switch_pipe(adev, PIPE_ID1);
3248                 /* maximum supported gfx ring is 2 */
3249                 ring = &adev->gfx.gfx_ring[1];
3250                 rb_bufsz = order_base_2(ring->ring_size / 8);
3251                 tmp = REG_SET_FIELD(0, CP_RB1_CNTL, RB_BUFSZ, rb_bufsz);
3252                 tmp = REG_SET_FIELD(tmp, CP_RB1_CNTL, RB_BLKSZ, rb_bufsz - 2);
3253                 WREG32_SOC15(GC, 0, regCP_RB1_CNTL, tmp);
3254                 /* Initialize the ring buffer's write pointers */
3255                 ring->wptr = 0;
3256                 WREG32_SOC15(GC, 0, regCP_RB1_WPTR, lower_32_bits(ring->wptr));
3257                 WREG32_SOC15(GC, 0, regCP_RB1_WPTR_HI, upper_32_bits(ring->wptr));
3258                 /* Set the wb address wether it's enabled or not */
3259                 rptr_addr = ring->rptr_gpu_addr;
3260                 WREG32_SOC15(GC, 0, regCP_RB1_RPTR_ADDR, lower_32_bits(rptr_addr));
3261                 WREG32_SOC15(GC, 0, regCP_RB1_RPTR_ADDR_HI, upper_32_bits(rptr_addr) &
3262                              CP_RB1_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK);
3263                 wptr_gpu_addr = ring->wptr_gpu_addr;
3264                 WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_ADDR_LO,
3265                              lower_32_bits(wptr_gpu_addr));
3266                 WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_ADDR_HI,
3267                              upper_32_bits(wptr_gpu_addr));
3268
3269                 mdelay(1);
3270                 WREG32_SOC15(GC, 0, regCP_RB1_CNTL, tmp);
3271
3272                 rb_addr = ring->gpu_addr >> 8;
3273                 WREG32_SOC15(GC, 0, regCP_RB1_BASE, rb_addr);
3274                 WREG32_SOC15(GC, 0, regCP_RB1_BASE_HI, upper_32_bits(rb_addr));
3275                 WREG32_SOC15(GC, 0, regCP_RB1_ACTIVE, 1);
3276
3277                 gfx_v11_0_cp_gfx_set_doorbell(adev, ring);
3278                 mutex_unlock(&adev->srbm_mutex);
3279         }
3280         /* Switch to pipe 0 */
3281         mutex_lock(&adev->srbm_mutex);
3282         gfx_v11_0_cp_gfx_switch_pipe(adev, PIPE_ID0);
3283         mutex_unlock(&adev->srbm_mutex);
3284
3285         /* start the ring */
3286         gfx_v11_0_cp_gfx_start(adev);
3287
3288         for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
3289                 ring = &adev->gfx.gfx_ring[i];
3290                 ring->sched.ready = true;
3291         }
3292
3293         return 0;
3294 }
3295
3296 static void gfx_v11_0_cp_compute_enable(struct amdgpu_device *adev, bool enable)
3297 {
3298         u32 data;
3299
3300         if (adev->gfx.rs64_enable) {
3301                 data = RREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL);
3302                 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_INVALIDATE_ICACHE,
3303                                                          enable ? 0 : 1);
3304                 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE0_RESET,
3305                                                          enable ? 0 : 1);
3306                 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE1_RESET,
3307                                                          enable ? 0 : 1);
3308                 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE2_RESET,
3309                                                          enable ? 0 : 1);
3310                 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE3_RESET,
3311                                                          enable ? 0 : 1);
3312                 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE0_ACTIVE,
3313                                                          enable ? 1 : 0);
3314                 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE1_ACTIVE,
3315                                                          enable ? 1 : 0);
3316                 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE2_ACTIVE,
3317                                                          enable ? 1 : 0);
3318                 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE3_ACTIVE,
3319                                                          enable ? 1 : 0);
3320                 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_HALT,
3321                                                          enable ? 0 : 1);
3322                 WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, data);
3323         } else {
3324                 data = RREG32_SOC15(GC, 0, regCP_MEC_CNTL);
3325
3326                 if (enable) {
3327                         data = REG_SET_FIELD(data, CP_MEC_CNTL, MEC_ME1_HALT, 0);
3328                         if (!adev->enable_mes_kiq)
3329                                 data = REG_SET_FIELD(data, CP_MEC_CNTL,
3330                                                      MEC_ME2_HALT, 0);
3331                 } else {
3332                         data = REG_SET_FIELD(data, CP_MEC_CNTL, MEC_ME1_HALT, 1);
3333                         data = REG_SET_FIELD(data, CP_MEC_CNTL, MEC_ME2_HALT, 1);
3334                 }
3335                 WREG32_SOC15(GC, 0, regCP_MEC_CNTL, data);
3336         }
3337
3338         adev->gfx.kiq.ring.sched.ready = enable;
3339
3340         udelay(50);
3341 }
3342
3343 static int gfx_v11_0_cp_compute_load_microcode(struct amdgpu_device *adev)
3344 {
3345         const struct gfx_firmware_header_v1_0 *mec_hdr;
3346         const __le32 *fw_data;
3347         unsigned i, fw_size;
3348         u32 *fw = NULL;
3349         int r;
3350
3351         if (!adev->gfx.mec_fw)
3352                 return -EINVAL;
3353
3354         gfx_v11_0_cp_compute_enable(adev, false);
3355
3356         mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
3357         amdgpu_ucode_print_gfx_hdr(&mec_hdr->header);
3358
3359         fw_data = (const __le32 *)
3360                 (adev->gfx.mec_fw->data +
3361                  le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
3362         fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes);
3363
3364         r = amdgpu_bo_create_reserved(adev, mec_hdr->header.ucode_size_bytes,
3365                                           PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
3366                                           &adev->gfx.mec.mec_fw_obj,
3367                                           &adev->gfx.mec.mec_fw_gpu_addr,
3368                                           (void **)&fw);
3369         if (r) {
3370                 dev_err(adev->dev, "(%d) failed to create mec fw bo\n", r);
3371                 gfx_v11_0_mec_fini(adev);
3372                 return r;
3373         }
3374
3375         memcpy(fw, fw_data, fw_size);
3376
3377         amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj);
3378         amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj);
3379
3380         gfx_v11_0_config_mec_cache(adev, adev->gfx.mec.mec_fw_gpu_addr);
3381
3382         /* MEC1 */
3383         WREG32_SOC15(GC, 0, regCP_MEC_ME1_UCODE_ADDR, 0);
3384
3385         for (i = 0; i < mec_hdr->jt_size; i++)
3386                 WREG32_SOC15(GC, 0, regCP_MEC_ME1_UCODE_DATA,
3387                              le32_to_cpup(fw_data + mec_hdr->jt_offset + i));
3388
3389         WREG32_SOC15(GC, 0, regCP_MEC_ME1_UCODE_ADDR, adev->gfx.mec_fw_version);
3390
3391         return 0;
3392 }
3393
3394 static int gfx_v11_0_cp_compute_load_microcode_rs64(struct amdgpu_device *adev)
3395 {
3396         const struct gfx_firmware_header_v2_0 *mec_hdr;
3397         const __le32 *fw_ucode, *fw_data;
3398         u32 tmp, fw_ucode_size, fw_data_size;
3399         u32 i, usec_timeout = 50000; /* Wait for 50 ms */
3400         u32 *fw_ucode_ptr, *fw_data_ptr;
3401         int r;
3402
3403         if (!adev->gfx.mec_fw)
3404                 return -EINVAL;
3405
3406         gfx_v11_0_cp_compute_enable(adev, false);
3407
3408         mec_hdr = (const struct gfx_firmware_header_v2_0 *)adev->gfx.mec_fw->data;
3409         amdgpu_ucode_print_gfx_hdr(&mec_hdr->header);
3410
3411         fw_ucode = (const __le32 *) (adev->gfx.mec_fw->data +
3412                                 le32_to_cpu(mec_hdr->ucode_offset_bytes));
3413         fw_ucode_size = le32_to_cpu(mec_hdr->ucode_size_bytes);
3414
3415         fw_data = (const __le32 *) (adev->gfx.mec_fw->data +
3416                                 le32_to_cpu(mec_hdr->data_offset_bytes));
3417         fw_data_size = le32_to_cpu(mec_hdr->data_size_bytes);
3418
3419         r = amdgpu_bo_create_reserved(adev, fw_ucode_size,
3420                                       64 * 1024,
3421                                       AMDGPU_GEM_DOMAIN_VRAM |
3422                                       AMDGPU_GEM_DOMAIN_GTT,
3423                                       &adev->gfx.mec.mec_fw_obj,
3424                                       &adev->gfx.mec.mec_fw_gpu_addr,
3425                                       (void **)&fw_ucode_ptr);
3426         if (r) {
3427                 dev_err(adev->dev, "(%d) failed to create mec fw ucode bo\n", r);
3428                 gfx_v11_0_mec_fini(adev);
3429                 return r;
3430         }
3431
3432         r = amdgpu_bo_create_reserved(adev, fw_data_size,
3433                                       64 * 1024,
3434                                       AMDGPU_GEM_DOMAIN_VRAM |
3435                                       AMDGPU_GEM_DOMAIN_GTT,
3436                                       &adev->gfx.mec.mec_fw_data_obj,
3437                                       &adev->gfx.mec.mec_fw_data_gpu_addr,
3438                                       (void **)&fw_data_ptr);
3439         if (r) {
3440                 dev_err(adev->dev, "(%d) failed to create mec fw ucode bo\n", r);
3441                 gfx_v11_0_mec_fini(adev);
3442                 return r;
3443         }
3444
3445         memcpy(fw_ucode_ptr, fw_ucode, fw_ucode_size);
3446         memcpy(fw_data_ptr, fw_data, fw_data_size);
3447
3448         amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj);
3449         amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_data_obj);
3450         amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj);
3451         amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_data_obj);
3452
3453         tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL);
3454         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0);
3455         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, EXE_DISABLE, 0);
3456         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0);
3457         WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL, tmp);
3458
3459         tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_BASE_CNTL);
3460         tmp = REG_SET_FIELD(tmp, CP_MEC_DC_BASE_CNTL, VMID, 0);
3461         tmp = REG_SET_FIELD(tmp, CP_MEC_DC_BASE_CNTL, CACHE_POLICY, 0);
3462         WREG32_SOC15(GC, 0, regCP_MEC_DC_BASE_CNTL, tmp);
3463
3464         mutex_lock(&adev->srbm_mutex);
3465         for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) {
3466                 soc21_grbm_select(adev, 1, i, 0, 0);
3467
3468                 WREG32_SOC15(GC, 0, regCP_MEC_MDBASE_LO, adev->gfx.mec.mec_fw_data_gpu_addr);
3469                 WREG32_SOC15(GC, 0, regCP_MEC_MDBASE_HI,
3470                      upper_32_bits(adev->gfx.mec.mec_fw_data_gpu_addr));
3471
3472                 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START,
3473                                         mec_hdr->ucode_start_addr_lo >> 2 |
3474                                         mec_hdr->ucode_start_addr_hi << 30);
3475                 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START_HI,
3476                                         mec_hdr->ucode_start_addr_hi >> 2);
3477
3478                 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_LO, adev->gfx.mec.mec_fw_gpu_addr);
3479                 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_HI,
3480                      upper_32_bits(adev->gfx.mec.mec_fw_gpu_addr));
3481         }
3482         mutex_unlock(&adev->srbm_mutex);
3483         soc21_grbm_select(adev, 0, 0, 0, 0);
3484
3485         /* Trigger an invalidation of the L1 instruction caches */
3486         tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL);
3487         tmp = REG_SET_FIELD(tmp, CP_MEC_DC_OP_CNTL, INVALIDATE_DCACHE, 1);
3488         WREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL, tmp);
3489
3490         /* Wait for invalidation complete */
3491         for (i = 0; i < usec_timeout; i++) {
3492                 tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL);
3493                 if (1 == REG_GET_FIELD(tmp, CP_MEC_DC_OP_CNTL,
3494                                        INVALIDATE_DCACHE_COMPLETE))
3495                         break;
3496                 udelay(1);
3497         }
3498
3499         if (i >= usec_timeout) {
3500                 dev_err(adev->dev, "failed to invalidate instruction cache\n");
3501                 return -EINVAL;
3502         }
3503
3504         /* Trigger an invalidation of the L1 instruction caches */
3505         tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL);
3506         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_OP_CNTL, INVALIDATE_CACHE, 1);
3507         WREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL, tmp);
3508
3509         /* Wait for invalidation complete */
3510         for (i = 0; i < usec_timeout; i++) {
3511                 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL);
3512                 if (1 == REG_GET_FIELD(tmp, CP_CPC_IC_OP_CNTL,
3513                                        INVALIDATE_CACHE_COMPLETE))
3514                         break;
3515                 udelay(1);
3516         }
3517
3518         if (i >= usec_timeout) {
3519                 dev_err(adev->dev, "failed to invalidate instruction cache\n");
3520                 return -EINVAL;
3521         }
3522
3523         return 0;
3524 }
3525
3526 static void gfx_v11_0_kiq_setting(struct amdgpu_ring *ring)
3527 {
3528         uint32_t tmp;
3529         struct amdgpu_device *adev = ring->adev;
3530
3531         /* tell RLC which is KIQ queue */
3532         tmp = RREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS);
3533         tmp &= 0xffffff00;
3534         tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue);
3535         WREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS, tmp);
3536         tmp |= 0x80;
3537         WREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS, tmp);
3538 }
3539
3540 static void gfx_v11_0_cp_set_doorbell_range(struct amdgpu_device *adev)
3541 {
3542         /* set graphics engine doorbell range */
3543         WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_RANGE_LOWER,
3544                      (adev->doorbell_index.gfx_ring0 * 2) << 2);
3545         WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_RANGE_UPPER,
3546                      (adev->doorbell_index.gfx_userqueue_end * 2) << 2);
3547
3548         /* set compute engine doorbell range */
3549         WREG32_SOC15(GC, 0, regCP_MEC_DOORBELL_RANGE_LOWER,
3550                      (adev->doorbell_index.kiq * 2) << 2);
3551         WREG32_SOC15(GC, 0, regCP_MEC_DOORBELL_RANGE_UPPER,
3552                      (adev->doorbell_index.userqueue_end * 2) << 2);
3553 }
3554
3555 static int gfx_v11_0_gfx_mqd_init(struct amdgpu_device *adev, void *m,
3556                                   struct amdgpu_mqd_prop *prop)
3557 {
3558         struct v11_gfx_mqd *mqd = m;
3559         uint64_t hqd_gpu_addr, wb_gpu_addr;
3560         uint32_t tmp;
3561         uint32_t rb_bufsz;
3562
3563         /* set up gfx hqd wptr */
3564         mqd->cp_gfx_hqd_wptr = 0;
3565         mqd->cp_gfx_hqd_wptr_hi = 0;
3566
3567         /* set the pointer to the MQD */
3568         mqd->cp_mqd_base_addr = prop->mqd_gpu_addr & 0xfffffffc;
3569         mqd->cp_mqd_base_addr_hi = upper_32_bits(prop->mqd_gpu_addr);
3570
3571         /* set up mqd control */
3572         tmp = RREG32_SOC15(GC, 0, regCP_GFX_MQD_CONTROL);
3573         tmp = REG_SET_FIELD(tmp, CP_GFX_MQD_CONTROL, VMID, 0);
3574         tmp = REG_SET_FIELD(tmp, CP_GFX_MQD_CONTROL, PRIV_STATE, 1);
3575         tmp = REG_SET_FIELD(tmp, CP_GFX_MQD_CONTROL, CACHE_POLICY, 0);
3576         mqd->cp_gfx_mqd_control = tmp;
3577
3578         /* set up gfx_hqd_vimd with 0x0 to indicate the ring buffer's vmid */
3579         tmp = RREG32_SOC15(GC, 0, regCP_GFX_HQD_VMID);
3580         tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_VMID, VMID, 0);
3581         mqd->cp_gfx_hqd_vmid = 0;
3582
3583         /* set up default queue priority level
3584          * 0x0 = low priority, 0x1 = high priority */
3585         tmp = RREG32_SOC15(GC, 0, regCP_GFX_HQD_QUEUE_PRIORITY);
3586         tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_QUEUE_PRIORITY, PRIORITY_LEVEL, 0);
3587         mqd->cp_gfx_hqd_queue_priority = tmp;
3588
3589         /* set up time quantum */
3590         tmp = RREG32_SOC15(GC, 0, regCP_GFX_HQD_QUANTUM);
3591         tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_QUANTUM, QUANTUM_EN, 1);
3592         mqd->cp_gfx_hqd_quantum = tmp;
3593
3594         /* set up gfx hqd base. this is similar as CP_RB_BASE */
3595         hqd_gpu_addr = prop->hqd_base_gpu_addr >> 8;
3596         mqd->cp_gfx_hqd_base = hqd_gpu_addr;
3597         mqd->cp_gfx_hqd_base_hi = upper_32_bits(hqd_gpu_addr);
3598
3599         /* set up hqd_rptr_addr/_hi, similar as CP_RB_RPTR */
3600         wb_gpu_addr = prop->rptr_gpu_addr;
3601         mqd->cp_gfx_hqd_rptr_addr = wb_gpu_addr & 0xfffffffc;
3602         mqd->cp_gfx_hqd_rptr_addr_hi =
3603                 upper_32_bits(wb_gpu_addr) & 0xffff;
3604
3605         /* set up rb_wptr_poll addr */
3606         wb_gpu_addr = prop->wptr_gpu_addr;
3607         mqd->cp_rb_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc;
3608         mqd->cp_rb_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
3609
3610         /* set up the gfx_hqd_control, similar as CP_RB0_CNTL */
3611         rb_bufsz = order_base_2(prop->queue_size / 4) - 1;
3612         tmp = RREG32_SOC15(GC, 0, regCP_GFX_HQD_CNTL);
3613         tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, RB_BUFSZ, rb_bufsz);
3614         tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, RB_BLKSZ, rb_bufsz - 2);
3615 #ifdef __BIG_ENDIAN
3616         tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, BUF_SWAP, 1);
3617 #endif
3618         mqd->cp_gfx_hqd_cntl = tmp;
3619
3620         /* set up cp_doorbell_control */
3621         tmp = RREG32_SOC15(GC, 0, regCP_RB_DOORBELL_CONTROL);
3622         if (prop->use_doorbell) {
3623                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3624                                     DOORBELL_OFFSET, prop->doorbell_index);
3625                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3626                                     DOORBELL_EN, 1);
3627         } else
3628                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3629                                     DOORBELL_EN, 0);
3630         mqd->cp_rb_doorbell_control = tmp;
3631
3632         /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
3633         mqd->cp_gfx_hqd_rptr = RREG32_SOC15(GC, 0, regCP_GFX_HQD_RPTR);
3634
3635         /* active the queue */
3636         mqd->cp_gfx_hqd_active = 1;
3637
3638         return 0;
3639 }
3640
3641 #ifdef BRING_UP_DEBUG
3642 static int gfx_v11_0_gfx_queue_init_register(struct amdgpu_ring *ring)
3643 {
3644         struct amdgpu_device *adev = ring->adev;
3645         struct v11_gfx_mqd *mqd = ring->mqd_ptr;
3646
3647         /* set mmCP_GFX_HQD_WPTR/_HI to 0 */
3648         WREG32_SOC15(GC, 0, regCP_GFX_HQD_WPTR, mqd->cp_gfx_hqd_wptr);
3649         WREG32_SOC15(GC, 0, regCP_GFX_HQD_WPTR_HI, mqd->cp_gfx_hqd_wptr_hi);
3650
3651         /* set GFX_MQD_BASE */
3652         WREG32_SOC15(GC, 0, regCP_MQD_BASE_ADDR, mqd->cp_mqd_base_addr);
3653         WREG32_SOC15(GC, 0, regCP_MQD_BASE_ADDR_HI, mqd->cp_mqd_base_addr_hi);
3654
3655         /* set GFX_MQD_CONTROL */
3656         WREG32_SOC15(GC, 0, regCP_GFX_MQD_CONTROL, mqd->cp_gfx_mqd_control);
3657
3658         /* set GFX_HQD_VMID to 0 */
3659         WREG32_SOC15(GC, 0, regCP_GFX_HQD_VMID, mqd->cp_gfx_hqd_vmid);
3660
3661         WREG32_SOC15(GC, 0, regCP_GFX_HQD_QUEUE_PRIORITY,
3662                         mqd->cp_gfx_hqd_queue_priority);
3663         WREG32_SOC15(GC, 0, regCP_GFX_HQD_QUANTUM, mqd->cp_gfx_hqd_quantum);
3664
3665         /* set GFX_HQD_BASE, similar as CP_RB_BASE */
3666         WREG32_SOC15(GC, 0, regCP_GFX_HQD_BASE, mqd->cp_gfx_hqd_base);
3667         WREG32_SOC15(GC, 0, regCP_GFX_HQD_BASE_HI, mqd->cp_gfx_hqd_base_hi);
3668
3669         /* set GFX_HQD_RPTR_ADDR, similar as CP_RB_RPTR */
3670         WREG32_SOC15(GC, 0, regCP_GFX_HQD_RPTR_ADDR, mqd->cp_gfx_hqd_rptr_addr);
3671         WREG32_SOC15(GC, 0, regCP_GFX_HQD_RPTR_ADDR_HI, mqd->cp_gfx_hqd_rptr_addr_hi);
3672
3673         /* set GFX_HQD_CNTL, similar as CP_RB_CNTL */
3674         WREG32_SOC15(GC, 0, regCP_GFX_HQD_CNTL, mqd->cp_gfx_hqd_cntl);
3675
3676         /* set RB_WPTR_POLL_ADDR */
3677         WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_ADDR_LO, mqd->cp_rb_wptr_poll_addr_lo);
3678         WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_ADDR_HI, mqd->cp_rb_wptr_poll_addr_hi);
3679
3680         /* set RB_DOORBELL_CONTROL */
3681         WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_CONTROL, mqd->cp_rb_doorbell_control);
3682
3683         /* active the queue */
3684         WREG32_SOC15(GC, 0, regCP_GFX_HQD_ACTIVE, mqd->cp_gfx_hqd_active);
3685
3686         return 0;
3687 }
3688 #endif
3689
3690 static int gfx_v11_0_gfx_init_queue(struct amdgpu_ring *ring)
3691 {
3692         struct amdgpu_device *adev = ring->adev;
3693         struct v11_gfx_mqd *mqd = ring->mqd_ptr;
3694         int mqd_idx = ring - &adev->gfx.gfx_ring[0];
3695
3696         if (!amdgpu_in_reset(adev) && !adev->in_suspend) {
3697                 memset((void *)mqd, 0, sizeof(*mqd));
3698                 mutex_lock(&adev->srbm_mutex);
3699                 soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
3700                 amdgpu_ring_init_mqd(ring);
3701 #ifdef BRING_UP_DEBUG
3702                 gfx_v11_0_gfx_queue_init_register(ring);
3703 #endif
3704                 soc21_grbm_select(adev, 0, 0, 0, 0);
3705                 mutex_unlock(&adev->srbm_mutex);
3706                 if (adev->gfx.me.mqd_backup[mqd_idx])
3707                         memcpy(adev->gfx.me.mqd_backup[mqd_idx], mqd, sizeof(*mqd));
3708         } else if (amdgpu_in_reset(adev)) {
3709                 /* reset mqd with the backup copy */
3710                 if (adev->gfx.me.mqd_backup[mqd_idx])
3711                         memcpy(mqd, adev->gfx.me.mqd_backup[mqd_idx], sizeof(*mqd));
3712                 /* reset the ring */
3713                 ring->wptr = 0;
3714                 *ring->wptr_cpu_addr = 0;
3715                 amdgpu_ring_clear_ring(ring);
3716 #ifdef BRING_UP_DEBUG
3717                 mutex_lock(&adev->srbm_mutex);
3718                 soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
3719                 gfx_v11_0_gfx_queue_init_register(ring);
3720                 soc21_grbm_select(adev, 0, 0, 0, 0);
3721                 mutex_unlock(&adev->srbm_mutex);
3722 #endif
3723         } else {
3724                 amdgpu_ring_clear_ring(ring);
3725         }
3726
3727         return 0;
3728 }
3729
3730 #ifndef BRING_UP_DEBUG
3731 static int gfx_v11_0_kiq_enable_kgq(struct amdgpu_device *adev)
3732 {
3733         struct amdgpu_kiq *kiq = &adev->gfx.kiq;
3734         struct amdgpu_ring *kiq_ring = &adev->gfx.kiq.ring;
3735         int r, i;
3736
3737         if (!kiq->pmf || !kiq->pmf->kiq_map_queues)
3738                 return -EINVAL;
3739
3740         r = amdgpu_ring_alloc(kiq_ring, kiq->pmf->map_queues_size *
3741                                         adev->gfx.num_gfx_rings);
3742         if (r) {
3743                 DRM_ERROR("Failed to lock KIQ (%d).\n", r);
3744                 return r;
3745         }
3746
3747         for (i = 0; i < adev->gfx.num_gfx_rings; i++)
3748                 kiq->pmf->kiq_map_queues(kiq_ring, &adev->gfx.gfx_ring[i]);
3749
3750         return amdgpu_ring_test_helper(kiq_ring);
3751 }
3752 #endif
3753
3754 static int gfx_v11_0_cp_async_gfx_ring_resume(struct amdgpu_device *adev)
3755 {
3756         int r, i;
3757         struct amdgpu_ring *ring;
3758
3759         for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
3760                 ring = &adev->gfx.gfx_ring[i];
3761
3762                 r = amdgpu_bo_reserve(ring->mqd_obj, false);
3763                 if (unlikely(r != 0))
3764                         goto done;
3765
3766                 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
3767                 if (!r) {
3768                         r = gfx_v11_0_gfx_init_queue(ring);
3769                         amdgpu_bo_kunmap(ring->mqd_obj);
3770                         ring->mqd_ptr = NULL;
3771                 }
3772                 amdgpu_bo_unreserve(ring->mqd_obj);
3773                 if (r)
3774                         goto done;
3775         }
3776 #ifndef BRING_UP_DEBUG
3777         r = gfx_v11_0_kiq_enable_kgq(adev);
3778         if (r)
3779                 goto done;
3780 #endif
3781         r = gfx_v11_0_cp_gfx_start(adev);
3782         if (r)
3783                 goto done;
3784
3785         for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
3786                 ring = &adev->gfx.gfx_ring[i];
3787                 ring->sched.ready = true;
3788         }
3789 done:
3790         return r;
3791 }
3792
3793 static int gfx_v11_0_compute_mqd_init(struct amdgpu_device *adev, void *m,
3794                                       struct amdgpu_mqd_prop *prop)
3795 {
3796         struct v11_compute_mqd *mqd = m;
3797         uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr;
3798         uint32_t tmp;
3799
3800         mqd->header = 0xC0310800;
3801         mqd->compute_pipelinestat_enable = 0x00000001;
3802         mqd->compute_static_thread_mgmt_se0 = 0xffffffff;
3803         mqd->compute_static_thread_mgmt_se1 = 0xffffffff;
3804         mqd->compute_static_thread_mgmt_se2 = 0xffffffff;
3805         mqd->compute_static_thread_mgmt_se3 = 0xffffffff;
3806         mqd->compute_misc_reserved = 0x00000007;
3807
3808         eop_base_addr = prop->eop_gpu_addr >> 8;
3809         mqd->cp_hqd_eop_base_addr_lo = eop_base_addr;
3810         mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr);
3811
3812         /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
3813         tmp = RREG32_SOC15(GC, 0, regCP_HQD_EOP_CONTROL);
3814         tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE,
3815                         (order_base_2(GFX11_MEC_HPD_SIZE / 4) - 1));
3816
3817         mqd->cp_hqd_eop_control = tmp;
3818
3819         /* enable doorbell? */
3820         tmp = RREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL);
3821
3822         if (prop->use_doorbell) {
3823                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3824                                     DOORBELL_OFFSET, prop->doorbell_index);
3825                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3826                                     DOORBELL_EN, 1);
3827                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3828                                     DOORBELL_SOURCE, 0);
3829                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3830                                     DOORBELL_HIT, 0);
3831         } else {
3832                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3833                                     DOORBELL_EN, 0);
3834         }
3835
3836         mqd->cp_hqd_pq_doorbell_control = tmp;
3837
3838         /* disable the queue if it's active */
3839         mqd->cp_hqd_dequeue_request = 0;
3840         mqd->cp_hqd_pq_rptr = 0;
3841         mqd->cp_hqd_pq_wptr_lo = 0;
3842         mqd->cp_hqd_pq_wptr_hi = 0;
3843
3844         /* set the pointer to the MQD */
3845         mqd->cp_mqd_base_addr_lo = prop->mqd_gpu_addr & 0xfffffffc;
3846         mqd->cp_mqd_base_addr_hi = upper_32_bits(prop->mqd_gpu_addr);
3847
3848         /* set MQD vmid to 0 */
3849         tmp = RREG32_SOC15(GC, 0, regCP_MQD_CONTROL);
3850         tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0);
3851         mqd->cp_mqd_control = tmp;
3852
3853         /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
3854         hqd_gpu_addr = prop->hqd_base_gpu_addr >> 8;
3855         mqd->cp_hqd_pq_base_lo = hqd_gpu_addr;
3856         mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
3857
3858         /* set up the HQD, this is similar to CP_RB0_CNTL */
3859         tmp = RREG32_SOC15(GC, 0, regCP_HQD_PQ_CONTROL);
3860         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE,
3861                             (order_base_2(prop->queue_size / 4) - 1));
3862         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE,
3863                             (order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1));
3864         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 0);
3865         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, TUNNEL_DISPATCH, 0);
3866         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1);
3867         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1);
3868         mqd->cp_hqd_pq_control = tmp;
3869
3870         /* set the wb address whether it's enabled or not */
3871         wb_gpu_addr = prop->rptr_gpu_addr;
3872         mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc;
3873         mqd->cp_hqd_pq_rptr_report_addr_hi =
3874                 upper_32_bits(wb_gpu_addr) & 0xffff;
3875
3876         /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
3877         wb_gpu_addr = prop->wptr_gpu_addr;
3878         mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc;
3879         mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
3880
3881         tmp = 0;
3882         /* enable the doorbell if requested */
3883         if (prop->use_doorbell) {
3884                 tmp = RREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL);
3885                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3886                                 DOORBELL_OFFSET, prop->doorbell_index);
3887
3888                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3889                                     DOORBELL_EN, 1);
3890                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3891                                     DOORBELL_SOURCE, 0);
3892                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3893                                     DOORBELL_HIT, 0);
3894         }
3895
3896         mqd->cp_hqd_pq_doorbell_control = tmp;
3897
3898         /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
3899         mqd->cp_hqd_pq_rptr = RREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR);
3900
3901         /* set the vmid for the queue */
3902         mqd->cp_hqd_vmid = 0;
3903
3904         tmp = RREG32_SOC15(GC, 0, regCP_HQD_PERSISTENT_STATE);
3905         tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x55);
3906         mqd->cp_hqd_persistent_state = tmp;
3907
3908         /* set MIN_IB_AVAIL_SIZE */
3909         tmp = RREG32_SOC15(GC, 0, regCP_HQD_IB_CONTROL);
3910         tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 3);
3911         mqd->cp_hqd_ib_control = tmp;
3912
3913         /* set static priority for a compute queue/ring */
3914         mqd->cp_hqd_pipe_priority = prop->hqd_pipe_priority;
3915         mqd->cp_hqd_queue_priority = prop->hqd_queue_priority;
3916
3917         mqd->cp_hqd_active = prop->hqd_active;
3918
3919         return 0;
3920 }
3921
3922 static int gfx_v11_0_kiq_init_register(struct amdgpu_ring *ring)
3923 {
3924         struct amdgpu_device *adev = ring->adev;
3925         struct v11_compute_mqd *mqd = ring->mqd_ptr;
3926         int j;
3927
3928         /* inactivate the queue */
3929         if (amdgpu_sriov_vf(adev))
3930                 WREG32_SOC15(GC, 0, regCP_HQD_ACTIVE, 0);
3931
3932         /* disable wptr polling */
3933         WREG32_FIELD15_PREREG(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0);
3934
3935         /* write the EOP addr */
3936         WREG32_SOC15(GC, 0, regCP_HQD_EOP_BASE_ADDR,
3937                mqd->cp_hqd_eop_base_addr_lo);
3938         WREG32_SOC15(GC, 0, regCP_HQD_EOP_BASE_ADDR_HI,
3939                mqd->cp_hqd_eop_base_addr_hi);
3940
3941         /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
3942         WREG32_SOC15(GC, 0, regCP_HQD_EOP_CONTROL,
3943                mqd->cp_hqd_eop_control);
3944
3945         /* enable doorbell? */
3946         WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL,
3947                mqd->cp_hqd_pq_doorbell_control);
3948
3949         /* disable the queue if it's active */
3950         if (RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) & 1) {
3951                 WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST, 1);
3952                 for (j = 0; j < adev->usec_timeout; j++) {
3953                         if (!(RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) & 1))
3954                                 break;
3955                         udelay(1);
3956                 }
3957                 WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST,
3958                        mqd->cp_hqd_dequeue_request);
3959                 WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR,
3960                        mqd->cp_hqd_pq_rptr);
3961                 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_LO,
3962                        mqd->cp_hqd_pq_wptr_lo);
3963                 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_HI,
3964                        mqd->cp_hqd_pq_wptr_hi);
3965         }
3966
3967         /* set the pointer to the MQD */
3968         WREG32_SOC15(GC, 0, regCP_MQD_BASE_ADDR,
3969                mqd->cp_mqd_base_addr_lo);
3970         WREG32_SOC15(GC, 0, regCP_MQD_BASE_ADDR_HI,
3971                mqd->cp_mqd_base_addr_hi);
3972
3973         /* set MQD vmid to 0 */
3974         WREG32_SOC15(GC, 0, regCP_MQD_CONTROL,
3975                mqd->cp_mqd_control);
3976
3977         /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
3978         WREG32_SOC15(GC, 0, regCP_HQD_PQ_BASE,
3979                mqd->cp_hqd_pq_base_lo);
3980         WREG32_SOC15(GC, 0, regCP_HQD_PQ_BASE_HI,
3981                mqd->cp_hqd_pq_base_hi);
3982
3983         /* set up the HQD, this is similar to CP_RB0_CNTL */
3984         WREG32_SOC15(GC, 0, regCP_HQD_PQ_CONTROL,
3985                mqd->cp_hqd_pq_control);
3986
3987         /* set the wb address whether it's enabled or not */
3988         WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR_REPORT_ADDR,
3989                 mqd->cp_hqd_pq_rptr_report_addr_lo);
3990         WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR_REPORT_ADDR_HI,
3991                 mqd->cp_hqd_pq_rptr_report_addr_hi);
3992
3993         /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
3994         WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR,
3995                mqd->cp_hqd_pq_wptr_poll_addr_lo);
3996         WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR_HI,
3997                mqd->cp_hqd_pq_wptr_poll_addr_hi);
3998
3999         /* enable the doorbell if requested */
4000         if (ring->use_doorbell) {
4001                 WREG32_SOC15(GC, 0, regCP_MEC_DOORBELL_RANGE_LOWER,
4002                         (adev->doorbell_index.kiq * 2) << 2);
4003                 WREG32_SOC15(GC, 0, regCP_MEC_DOORBELL_RANGE_UPPER,
4004                         (adev->doorbell_index.userqueue_end * 2) << 2);
4005         }
4006
4007         WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL,
4008                mqd->cp_hqd_pq_doorbell_control);
4009
4010         /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
4011         WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_LO,
4012                mqd->cp_hqd_pq_wptr_lo);
4013         WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_HI,
4014                mqd->cp_hqd_pq_wptr_hi);
4015
4016         /* set the vmid for the queue */
4017         WREG32_SOC15(GC, 0, regCP_HQD_VMID, mqd->cp_hqd_vmid);
4018
4019         WREG32_SOC15(GC, 0, regCP_HQD_PERSISTENT_STATE,
4020                mqd->cp_hqd_persistent_state);
4021
4022         /* activate the queue */
4023         WREG32_SOC15(GC, 0, regCP_HQD_ACTIVE,
4024                mqd->cp_hqd_active);
4025
4026         if (ring->use_doorbell)
4027                 WREG32_FIELD15_PREREG(GC, 0, CP_PQ_STATUS, DOORBELL_ENABLE, 1);
4028
4029         return 0;
4030 }
4031
4032 static int gfx_v11_0_kiq_init_queue(struct amdgpu_ring *ring)
4033 {
4034         struct amdgpu_device *adev = ring->adev;
4035         struct v11_compute_mqd *mqd = ring->mqd_ptr;
4036         int mqd_idx = AMDGPU_MAX_COMPUTE_RINGS;
4037
4038         gfx_v11_0_kiq_setting(ring);
4039
4040         if (amdgpu_in_reset(adev)) { /* for GPU_RESET case */
4041                 /* reset MQD to a clean status */
4042                 if (adev->gfx.mec.mqd_backup[mqd_idx])
4043                         memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(*mqd));
4044
4045                 /* reset ring buffer */
4046                 ring->wptr = 0;
4047                 amdgpu_ring_clear_ring(ring);
4048
4049                 mutex_lock(&adev->srbm_mutex);
4050                 soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
4051                 gfx_v11_0_kiq_init_register(ring);
4052                 soc21_grbm_select(adev, 0, 0, 0, 0);
4053                 mutex_unlock(&adev->srbm_mutex);
4054         } else {
4055                 memset((void *)mqd, 0, sizeof(*mqd));
4056                 if (amdgpu_sriov_vf(adev) && adev->in_suspend)
4057                         amdgpu_ring_clear_ring(ring);
4058                 mutex_lock(&adev->srbm_mutex);
4059                 soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
4060                 amdgpu_ring_init_mqd(ring);
4061                 gfx_v11_0_kiq_init_register(ring);
4062                 soc21_grbm_select(adev, 0, 0, 0, 0);
4063                 mutex_unlock(&adev->srbm_mutex);
4064
4065                 if (adev->gfx.mec.mqd_backup[mqd_idx])
4066                         memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(*mqd));
4067         }
4068
4069         return 0;
4070 }
4071
4072 static int gfx_v11_0_kcq_init_queue(struct amdgpu_ring *ring)
4073 {
4074         struct amdgpu_device *adev = ring->adev;
4075         struct v11_compute_mqd *mqd = ring->mqd_ptr;
4076         int mqd_idx = ring - &adev->gfx.compute_ring[0];
4077
4078         if (!amdgpu_in_reset(adev) && !adev->in_suspend) {
4079                 memset((void *)mqd, 0, sizeof(*mqd));
4080                 mutex_lock(&adev->srbm_mutex);
4081                 soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
4082                 amdgpu_ring_init_mqd(ring);
4083                 soc21_grbm_select(adev, 0, 0, 0, 0);
4084                 mutex_unlock(&adev->srbm_mutex);
4085
4086                 if (adev->gfx.mec.mqd_backup[mqd_idx])
4087                         memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(*mqd));
4088         } else if (amdgpu_in_reset(adev)) { /* for GPU_RESET case */
4089                 /* reset MQD to a clean status */
4090                 if (adev->gfx.mec.mqd_backup[mqd_idx])
4091                         memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(*mqd));
4092
4093                 /* reset ring buffer */
4094                 ring->wptr = 0;
4095                 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, 0);
4096                 amdgpu_ring_clear_ring(ring);
4097         } else {
4098                 amdgpu_ring_clear_ring(ring);
4099         }
4100
4101         return 0;
4102 }
4103
4104 static int gfx_v11_0_kiq_resume(struct amdgpu_device *adev)
4105 {
4106         struct amdgpu_ring *ring;
4107         int r;
4108
4109         ring = &adev->gfx.kiq.ring;
4110
4111         r = amdgpu_bo_reserve(ring->mqd_obj, false);
4112         if (unlikely(r != 0))
4113                 return r;
4114
4115         r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
4116         if (unlikely(r != 0)) {
4117                 amdgpu_bo_unreserve(ring->mqd_obj);
4118                 return r;
4119         }
4120
4121         gfx_v11_0_kiq_init_queue(ring);
4122         amdgpu_bo_kunmap(ring->mqd_obj);
4123         ring->mqd_ptr = NULL;
4124         amdgpu_bo_unreserve(ring->mqd_obj);
4125         ring->sched.ready = true;
4126         return 0;
4127 }
4128
4129 static int gfx_v11_0_kcq_resume(struct amdgpu_device *adev)
4130 {
4131         struct amdgpu_ring *ring = NULL;
4132         int r = 0, i;
4133
4134         if (!amdgpu_async_gfx_ring)
4135                 gfx_v11_0_cp_compute_enable(adev, true);
4136
4137         for (i = 0; i < adev->gfx.num_compute_rings; i++) {
4138                 ring = &adev->gfx.compute_ring[i];
4139
4140                 r = amdgpu_bo_reserve(ring->mqd_obj, false);
4141                 if (unlikely(r != 0))
4142                         goto done;
4143                 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
4144                 if (!r) {
4145                         r = gfx_v11_0_kcq_init_queue(ring);
4146                         amdgpu_bo_kunmap(ring->mqd_obj);
4147                         ring->mqd_ptr = NULL;
4148                 }
4149                 amdgpu_bo_unreserve(ring->mqd_obj);
4150                 if (r)
4151                         goto done;
4152         }
4153
4154         r = amdgpu_gfx_enable_kcq(adev);
4155 done:
4156         return r;
4157 }
4158
4159 static int gfx_v11_0_cp_resume(struct amdgpu_device *adev)
4160 {
4161         int r, i;
4162         struct amdgpu_ring *ring;
4163
4164         if (!(adev->flags & AMD_IS_APU))
4165                 gfx_v11_0_enable_gui_idle_interrupt(adev, false);
4166
4167         if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
4168                 /* legacy firmware loading */
4169                 r = gfx_v11_0_cp_gfx_load_microcode(adev);
4170                 if (r)
4171                         return r;
4172
4173                 if (adev->gfx.rs64_enable)
4174                         r = gfx_v11_0_cp_compute_load_microcode_rs64(adev);
4175                 else
4176                         r = gfx_v11_0_cp_compute_load_microcode(adev);
4177                 if (r)
4178                         return r;
4179         }
4180
4181         gfx_v11_0_cp_set_doorbell_range(adev);
4182
4183         if (amdgpu_async_gfx_ring) {
4184                 gfx_v11_0_cp_compute_enable(adev, true);
4185                 gfx_v11_0_cp_gfx_enable(adev, true);
4186         }
4187
4188         if (adev->enable_mes_kiq && adev->mes.kiq_hw_init)
4189                 r = amdgpu_mes_kiq_hw_init(adev);
4190         else
4191                 r = gfx_v11_0_kiq_resume(adev);
4192         if (r)
4193                 return r;
4194
4195         r = gfx_v11_0_kcq_resume(adev);
4196         if (r)
4197                 return r;
4198
4199         if (!amdgpu_async_gfx_ring) {
4200                 r = gfx_v11_0_cp_gfx_resume(adev);
4201                 if (r)
4202                         return r;
4203         } else {
4204                 r = gfx_v11_0_cp_async_gfx_ring_resume(adev);
4205                 if (r)
4206                         return r;
4207         }
4208
4209         for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
4210                 ring = &adev->gfx.gfx_ring[i];
4211                 r = amdgpu_ring_test_helper(ring);
4212                 if (r)
4213                         return r;
4214         }
4215
4216         for (i = 0; i < adev->gfx.num_compute_rings; i++) {
4217                 ring = &adev->gfx.compute_ring[i];
4218                 r = amdgpu_ring_test_helper(ring);
4219                 if (r)
4220                         return r;
4221         }
4222
4223         return 0;
4224 }
4225
4226 static void gfx_v11_0_cp_enable(struct amdgpu_device *adev, bool enable)
4227 {
4228         gfx_v11_0_cp_gfx_enable(adev, enable);
4229         gfx_v11_0_cp_compute_enable(adev, enable);
4230 }
4231
4232 static int gfx_v11_0_gfxhub_enable(struct amdgpu_device *adev)
4233 {
4234         int r;
4235         bool value;
4236
4237         r = adev->gfxhub.funcs->gart_enable(adev);
4238         if (r)
4239                 return r;
4240
4241         adev->hdp.funcs->flush_hdp(adev, NULL);
4242
4243         value = (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS) ?
4244                 false : true;
4245
4246         adev->gfxhub.funcs->set_fault_enable_default(adev, value);
4247         amdgpu_gmc_flush_gpu_tlb(adev, 0, AMDGPU_GFXHUB_0, 0);
4248
4249         return 0;
4250 }
4251
4252 static void gfx_v11_0_select_cp_fw_arch(struct amdgpu_device *adev)
4253 {
4254         u32 tmp;
4255
4256         /* select RS64 */
4257         if (adev->gfx.rs64_enable) {
4258                 tmp = RREG32_SOC15(GC, 0, regCP_GFX_CNTL);
4259                 tmp = REG_SET_FIELD(tmp, CP_GFX_CNTL, ENGINE_SEL, 1);
4260                 WREG32_SOC15(GC, 0, regCP_GFX_CNTL, tmp);
4261
4262                 tmp = RREG32_SOC15(GC, 0, regCP_MEC_ISA_CNTL);
4263                 tmp = REG_SET_FIELD(tmp, CP_MEC_ISA_CNTL, ISA_MODE, 1);
4264                 WREG32_SOC15(GC, 0, regCP_MEC_ISA_CNTL, tmp);
4265         }
4266
4267         if (amdgpu_emu_mode == 1)
4268                 msleep(100);
4269 }
4270
4271 static int get_gb_addr_config(struct amdgpu_device * adev)
4272 {
4273         u32 gb_addr_config;
4274
4275         gb_addr_config = RREG32_SOC15(GC, 0, regGB_ADDR_CONFIG);
4276         if (gb_addr_config == 0)
4277                 return -EINVAL;
4278
4279         adev->gfx.config.gb_addr_config_fields.num_pkrs =
4280                 1 << REG_GET_FIELD(gb_addr_config, GB_ADDR_CONFIG, NUM_PKRS);
4281
4282         adev->gfx.config.gb_addr_config = gb_addr_config;
4283
4284         adev->gfx.config.gb_addr_config_fields.num_pipes = 1 <<
4285                         REG_GET_FIELD(adev->gfx.config.gb_addr_config,
4286                                       GB_ADDR_CONFIG, NUM_PIPES);
4287
4288         adev->gfx.config.max_tile_pipes =
4289                 adev->gfx.config.gb_addr_config_fields.num_pipes;
4290
4291         adev->gfx.config.gb_addr_config_fields.max_compress_frags = 1 <<
4292                         REG_GET_FIELD(adev->gfx.config.gb_addr_config,
4293                                       GB_ADDR_CONFIG, MAX_COMPRESSED_FRAGS);
4294         adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 1 <<
4295                         REG_GET_FIELD(adev->gfx.config.gb_addr_config,
4296                                       GB_ADDR_CONFIG, NUM_RB_PER_SE);
4297         adev->gfx.config.gb_addr_config_fields.num_se = 1 <<
4298                         REG_GET_FIELD(adev->gfx.config.gb_addr_config,
4299                                       GB_ADDR_CONFIG, NUM_SHADER_ENGINES);
4300         adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 1 << (8 +
4301                         REG_GET_FIELD(adev->gfx.config.gb_addr_config,
4302                                       GB_ADDR_CONFIG, PIPE_INTERLEAVE_SIZE));
4303
4304         return 0;
4305 }
4306
4307 static void gfx_v11_0_disable_gpa_mode(struct amdgpu_device *adev)
4308 {
4309         uint32_t data;
4310
4311         data = RREG32_SOC15(GC, 0, regCPC_PSP_DEBUG);
4312         data |= CPC_PSP_DEBUG__GPA_OVERRIDE_MASK;
4313         WREG32_SOC15(GC, 0, regCPC_PSP_DEBUG, data);
4314
4315         data = RREG32_SOC15(GC, 0, regCPG_PSP_DEBUG);
4316         data |= CPG_PSP_DEBUG__GPA_OVERRIDE_MASK;
4317         WREG32_SOC15(GC, 0, regCPG_PSP_DEBUG, data);
4318 }
4319
4320 static int gfx_v11_0_hw_init(void *handle)
4321 {
4322         int r;
4323         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4324
4325         if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) {
4326                 if (adev->gfx.imu.funcs) {
4327                         /* RLC autoload sequence 1: Program rlc ram */
4328                         if (adev->gfx.imu.funcs->program_rlc_ram)
4329                                 adev->gfx.imu.funcs->program_rlc_ram(adev);
4330                 }
4331                 /* rlc autoload firmware */
4332                 r = gfx_v11_0_rlc_backdoor_autoload_enable(adev);
4333                 if (r)
4334                         return r;
4335         } else {
4336                 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
4337                         if (adev->gfx.imu.funcs && (amdgpu_dpm > 0)) {
4338                                 if (adev->gfx.imu.funcs->load_microcode)
4339                                         adev->gfx.imu.funcs->load_microcode(adev);
4340                                 if (adev->gfx.imu.funcs->setup_imu)
4341                                         adev->gfx.imu.funcs->setup_imu(adev);
4342                                 if (adev->gfx.imu.funcs->start_imu)
4343                                         adev->gfx.imu.funcs->start_imu(adev);
4344                         }
4345
4346                         /* disable gpa mode in backdoor loading */
4347                         gfx_v11_0_disable_gpa_mode(adev);
4348                 }
4349         }
4350
4351         if ((adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) ||
4352             (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP)) {
4353                 r = gfx_v11_0_wait_for_rlc_autoload_complete(adev);
4354                 if (r) {
4355                         dev_err(adev->dev, "(%d) failed to wait rlc autoload complete\n", r);
4356                         return r;
4357                 }
4358         }
4359
4360         adev->gfx.is_poweron = true;
4361
4362         if(get_gb_addr_config(adev))
4363                 DRM_WARN("Invalid gb_addr_config !\n");
4364
4365         if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP &&
4366             adev->gfx.rs64_enable)
4367                 gfx_v11_0_config_gfx_rs64(adev);
4368
4369         r = gfx_v11_0_gfxhub_enable(adev);
4370         if (r)
4371                 return r;
4372
4373         if (!amdgpu_emu_mode)
4374                 gfx_v11_0_init_golden_registers(adev);
4375
4376         if ((adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) ||
4377             (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO && amdgpu_dpm == 1)) {
4378                 /**
4379                  * For gfx 11, rlc firmware loading relies on smu firmware is
4380                  * loaded firstly, so in direct type, it has to load smc ucode
4381                  * here before rlc.
4382                  */
4383                 if (!(adev->flags & AMD_IS_APU)) {
4384                         r = amdgpu_pm_load_smu_firmware(adev, NULL);
4385                         if (r)
4386                                 return r;
4387                 }
4388         }
4389
4390         gfx_v11_0_constants_init(adev);
4391
4392         if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP)
4393                 gfx_v11_0_select_cp_fw_arch(adev);
4394
4395         if (adev->nbio.funcs->gc_doorbell_init)
4396                 adev->nbio.funcs->gc_doorbell_init(adev);
4397
4398         r = gfx_v11_0_rlc_resume(adev);
4399         if (r)
4400                 return r;
4401
4402         /*
4403          * init golden registers and rlc resume may override some registers,
4404          * reconfig them here
4405          */
4406         gfx_v11_0_tcp_harvest(adev);
4407
4408         r = gfx_v11_0_cp_resume(adev);
4409         if (r)
4410                 return r;
4411
4412         return r;
4413 }
4414
4415 #ifndef BRING_UP_DEBUG
4416 static int gfx_v11_0_kiq_disable_kgq(struct amdgpu_device *adev)
4417 {
4418         struct amdgpu_kiq *kiq = &adev->gfx.kiq;
4419         struct amdgpu_ring *kiq_ring = &kiq->ring;
4420         int i, r = 0;
4421
4422         if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues)
4423                 return -EINVAL;
4424
4425         if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size *
4426                                         adev->gfx.num_gfx_rings))
4427                 return -ENOMEM;
4428
4429         for (i = 0; i < adev->gfx.num_gfx_rings; i++)
4430                 kiq->pmf->kiq_unmap_queues(kiq_ring, &adev->gfx.gfx_ring[i],
4431                                            PREEMPT_QUEUES, 0, 0);
4432
4433         if (adev->gfx.kiq.ring.sched.ready)
4434                 r = amdgpu_ring_test_helper(kiq_ring);
4435
4436         return r;
4437 }
4438 #endif
4439
4440 static int gfx_v11_0_hw_fini(void *handle)
4441 {
4442         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4443         int r;
4444
4445         amdgpu_irq_put(adev, &adev->gfx.cp_ecc_error_irq, 0);
4446         amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
4447         amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
4448
4449         if (!adev->no_hw_access) {
4450 #ifndef BRING_UP_DEBUG
4451                 if (amdgpu_async_gfx_ring) {
4452                         r = gfx_v11_0_kiq_disable_kgq(adev);
4453                         if (r)
4454                                 DRM_ERROR("KGQ disable failed\n");
4455                 }
4456 #endif
4457                 if (amdgpu_gfx_disable_kcq(adev))
4458                         DRM_ERROR("KCQ disable failed\n");
4459
4460                 amdgpu_mes_kiq_hw_fini(adev);
4461         }
4462
4463         if (amdgpu_sriov_vf(adev))
4464                 /* Remove the steps disabling CPG and clearing KIQ position,
4465                  * so that CP could perform IDLE-SAVE during switch. Those
4466                  * steps are necessary to avoid a DMAR error in gfx9 but it is
4467                  * not reproduced on gfx11.
4468                  */
4469                 return 0;
4470
4471         gfx_v11_0_cp_enable(adev, false);
4472         gfx_v11_0_enable_gui_idle_interrupt(adev, false);
4473
4474         adev->gfxhub.funcs->gart_disable(adev);
4475
4476         adev->gfx.is_poweron = false;
4477
4478         return 0;
4479 }
4480
4481 static int gfx_v11_0_suspend(void *handle)
4482 {
4483         return gfx_v11_0_hw_fini(handle);
4484 }
4485
4486 static int gfx_v11_0_resume(void *handle)
4487 {
4488         return gfx_v11_0_hw_init(handle);
4489 }
4490
4491 static bool gfx_v11_0_is_idle(void *handle)
4492 {
4493         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4494
4495         if (REG_GET_FIELD(RREG32_SOC15(GC, 0, regGRBM_STATUS),
4496                                 GRBM_STATUS, GUI_ACTIVE))
4497                 return false;
4498         else
4499                 return true;
4500 }
4501
4502 static int gfx_v11_0_wait_for_idle(void *handle)
4503 {
4504         unsigned i;
4505         u32 tmp;
4506         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4507
4508         for (i = 0; i < adev->usec_timeout; i++) {
4509                 /* read MC_STATUS */
4510                 tmp = RREG32_SOC15(GC, 0, regGRBM_STATUS) &
4511                         GRBM_STATUS__GUI_ACTIVE_MASK;
4512
4513                 if (!REG_GET_FIELD(tmp, GRBM_STATUS, GUI_ACTIVE))
4514                         return 0;
4515                 udelay(1);
4516         }
4517         return -ETIMEDOUT;
4518 }
4519
4520 static int gfx_v11_0_soft_reset(void *handle)
4521 {
4522         u32 grbm_soft_reset = 0;
4523         u32 tmp;
4524         int i, j, k;
4525         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4526
4527         tmp = RREG32_SOC15(GC, 0, regCP_INT_CNTL);
4528         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CMP_BUSY_INT_ENABLE, 0);
4529         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CNTX_BUSY_INT_ENABLE, 0);
4530         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CNTX_EMPTY_INT_ENABLE, 0);
4531         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, GFX_IDLE_INT_ENABLE, 0);
4532         WREG32_SOC15(GC, 0, regCP_INT_CNTL, tmp);
4533
4534         gfx_v11_0_set_safe_mode(adev);
4535
4536         for (i = 0; i < adev->gfx.mec.num_mec; ++i) {
4537                 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {
4538                         for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {
4539                                 tmp = RREG32_SOC15(GC, 0, regGRBM_GFX_CNTL);
4540                                 tmp = REG_SET_FIELD(tmp, GRBM_GFX_CNTL, MEID, i);
4541                                 tmp = REG_SET_FIELD(tmp, GRBM_GFX_CNTL, QUEUEID, j);
4542                                 tmp = REG_SET_FIELD(tmp, GRBM_GFX_CNTL, PIPEID, k);
4543                                 WREG32_SOC15(GC, 0, regGRBM_GFX_CNTL, tmp);
4544
4545                                 WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST, 0x2);
4546                                 WREG32_SOC15(GC, 0, regSPI_COMPUTE_QUEUE_RESET, 0x1);
4547                         }
4548                 }
4549         }
4550         for (i = 0; i < adev->gfx.me.num_me; ++i) {
4551                 for (j = 0; j < adev->gfx.me.num_queue_per_pipe; j++) {
4552                         for (k = 0; k < adev->gfx.me.num_pipe_per_me; k++) {
4553                                 tmp = RREG32_SOC15(GC, 0, regGRBM_GFX_CNTL);
4554                                 tmp = REG_SET_FIELD(tmp, GRBM_GFX_CNTL, MEID, i);
4555                                 tmp = REG_SET_FIELD(tmp, GRBM_GFX_CNTL, QUEUEID, j);
4556                                 tmp = REG_SET_FIELD(tmp, GRBM_GFX_CNTL, PIPEID, k);
4557                                 WREG32_SOC15(GC, 0, regGRBM_GFX_CNTL, tmp);
4558
4559                                 WREG32_SOC15(GC, 0, regCP_GFX_HQD_DEQUEUE_REQUEST, 0x1);
4560                         }
4561                 }
4562         }
4563
4564         WREG32_SOC15(GC, 0, regCP_VMID_RESET, 0xfffffffe);
4565
4566         // Read CP_VMID_RESET register three times.
4567         // to get sufficient time for GFX_HQD_ACTIVE reach 0
4568         RREG32_SOC15(GC, 0, regCP_VMID_RESET);
4569         RREG32_SOC15(GC, 0, regCP_VMID_RESET);
4570         RREG32_SOC15(GC, 0, regCP_VMID_RESET);
4571
4572         for (i = 0; i < adev->usec_timeout; i++) {
4573                 if (!RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) &&
4574                     !RREG32_SOC15(GC, 0, regCP_GFX_HQD_ACTIVE))
4575                         break;
4576                 udelay(1);
4577         }
4578         if (i >= adev->usec_timeout) {
4579                 printk("Failed to wait all pipes clean\n");
4580                 return -EINVAL;
4581         }
4582
4583         /**********  trigger soft reset  ***********/
4584         grbm_soft_reset = RREG32_SOC15(GC, 0, regGRBM_SOFT_RESET);
4585         grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
4586                                         SOFT_RESET_CP, 1);
4587         grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
4588                                         SOFT_RESET_GFX, 1);
4589         grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
4590                                         SOFT_RESET_CPF, 1);
4591         grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
4592                                         SOFT_RESET_CPC, 1);
4593         grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
4594                                         SOFT_RESET_CPG, 1);
4595         WREG32_SOC15(GC, 0, regGRBM_SOFT_RESET, grbm_soft_reset);
4596         /**********  exit soft reset  ***********/
4597         grbm_soft_reset = RREG32_SOC15(GC, 0, regGRBM_SOFT_RESET);
4598         grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
4599                                         SOFT_RESET_CP, 0);
4600         grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
4601                                         SOFT_RESET_GFX, 0);
4602         grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
4603                                         SOFT_RESET_CPF, 0);
4604         grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
4605                                         SOFT_RESET_CPC, 0);
4606         grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
4607                                         SOFT_RESET_CPG, 0);
4608         WREG32_SOC15(GC, 0, regGRBM_SOFT_RESET, grbm_soft_reset);
4609
4610         tmp = RREG32_SOC15(GC, 0, regCP_SOFT_RESET_CNTL);
4611         tmp = REG_SET_FIELD(tmp, CP_SOFT_RESET_CNTL, CMP_HQD_REG_RESET, 0x1);
4612         WREG32_SOC15(GC, 0, regCP_SOFT_RESET_CNTL, tmp);
4613
4614         WREG32_SOC15(GC, 0, regCP_ME_CNTL, 0x0);
4615         WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, 0x0);
4616
4617         for (i = 0; i < adev->usec_timeout; i++) {
4618                 if (!RREG32_SOC15(GC, 0, regCP_VMID_RESET))
4619                         break;
4620                 udelay(1);
4621         }
4622         if (i >= adev->usec_timeout) {
4623                 printk("Failed to wait CP_VMID_RESET to 0\n");
4624                 return -EINVAL;
4625         }
4626
4627         tmp = RREG32_SOC15(GC, 0, regCP_INT_CNTL);
4628         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CMP_BUSY_INT_ENABLE, 1);
4629         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CNTX_BUSY_INT_ENABLE, 1);
4630         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CNTX_EMPTY_INT_ENABLE, 1);
4631         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, GFX_IDLE_INT_ENABLE, 1);
4632         WREG32_SOC15(GC, 0, regCP_INT_CNTL, tmp);
4633
4634         gfx_v11_0_unset_safe_mode(adev);
4635
4636         return gfx_v11_0_cp_resume(adev);
4637 }
4638
4639 static bool gfx_v11_0_check_soft_reset(void *handle)
4640 {
4641         int i, r;
4642         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4643         struct amdgpu_ring *ring;
4644         long tmo = msecs_to_jiffies(1000);
4645
4646         for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
4647                 ring = &adev->gfx.gfx_ring[i];
4648                 r = amdgpu_ring_test_ib(ring, tmo);
4649                 if (r)
4650                         return true;
4651         }
4652
4653         for (i = 0; i < adev->gfx.num_compute_rings; i++) {
4654                 ring = &adev->gfx.compute_ring[i];
4655                 r = amdgpu_ring_test_ib(ring, tmo);
4656                 if (r)
4657                         return true;
4658         }
4659
4660         return false;
4661 }
4662
4663 static int gfx_v11_0_post_soft_reset(void *handle)
4664 {
4665         /**
4666          * GFX soft reset will impact MES, need resume MES when do GFX soft reset
4667          */
4668         return amdgpu_mes_resume((struct amdgpu_device *)handle);
4669 }
4670
4671 static uint64_t gfx_v11_0_get_gpu_clock_counter(struct amdgpu_device *adev)
4672 {
4673         uint64_t clock;
4674
4675         amdgpu_gfx_off_ctrl(adev, false);
4676         mutex_lock(&adev->gfx.gpu_clock_mutex);
4677         clock = (uint64_t)RREG32_SOC15(SMUIO, 0, regGOLDEN_TSC_COUNT_LOWER) |
4678                 ((uint64_t)RREG32_SOC15(SMUIO, 0, regGOLDEN_TSC_COUNT_UPPER) << 32ULL);
4679         mutex_unlock(&adev->gfx.gpu_clock_mutex);
4680         amdgpu_gfx_off_ctrl(adev, true);
4681         return clock;
4682 }
4683
4684 static void gfx_v11_0_ring_emit_gds_switch(struct amdgpu_ring *ring,
4685                                            uint32_t vmid,
4686                                            uint32_t gds_base, uint32_t gds_size,
4687                                            uint32_t gws_base, uint32_t gws_size,
4688                                            uint32_t oa_base, uint32_t oa_size)
4689 {
4690         struct amdgpu_device *adev = ring->adev;
4691
4692         /* GDS Base */
4693         gfx_v11_0_write_data_to_reg(ring, 0, false,
4694                                     SOC15_REG_OFFSET(GC, 0, regGDS_VMID0_BASE) + 2 * vmid,
4695                                     gds_base);
4696
4697         /* GDS Size */
4698         gfx_v11_0_write_data_to_reg(ring, 0, false,
4699                                     SOC15_REG_OFFSET(GC, 0, regGDS_VMID0_SIZE) + 2 * vmid,
4700                                     gds_size);
4701
4702         /* GWS */
4703         gfx_v11_0_write_data_to_reg(ring, 0, false,
4704                                     SOC15_REG_OFFSET(GC, 0, regGDS_GWS_VMID0) + vmid,
4705                                     gws_size << GDS_GWS_VMID0__SIZE__SHIFT | gws_base);
4706
4707         /* OA */
4708         gfx_v11_0_write_data_to_reg(ring, 0, false,
4709                                     SOC15_REG_OFFSET(GC, 0, regGDS_OA_VMID0) + vmid,
4710                                     (1 << (oa_size + oa_base)) - (1 << oa_base));
4711 }
4712
4713 static int gfx_v11_0_early_init(void *handle)
4714 {
4715         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4716
4717         adev->gfx.funcs = &gfx_v11_0_gfx_funcs;
4718
4719         adev->gfx.num_gfx_rings = GFX11_NUM_GFX_RINGS;
4720         adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev),
4721                                           AMDGPU_MAX_COMPUTE_RINGS);
4722
4723         gfx_v11_0_set_kiq_pm4_funcs(adev);
4724         gfx_v11_0_set_ring_funcs(adev);
4725         gfx_v11_0_set_irq_funcs(adev);
4726         gfx_v11_0_set_gds_init(adev);
4727         gfx_v11_0_set_rlc_funcs(adev);
4728         gfx_v11_0_set_mqd_funcs(adev);
4729         gfx_v11_0_set_imu_funcs(adev);
4730
4731         gfx_v11_0_init_rlcg_reg_access_ctrl(adev);
4732
4733         return gfx_v11_0_init_microcode(adev);
4734 }
4735
4736 static int gfx_v11_0_ras_late_init(void *handle)
4737 {
4738         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4739         struct ras_common_if *gfx_common_if;
4740         int ret;
4741
4742         gfx_common_if = kzalloc(sizeof(struct ras_common_if), GFP_KERNEL);
4743         if (!gfx_common_if)
4744                 return -ENOMEM;
4745
4746         gfx_common_if->block = AMDGPU_RAS_BLOCK__GFX;
4747
4748         ret = amdgpu_ras_feature_enable(adev, gfx_common_if, true);
4749         if (ret)
4750                 dev_warn(adev->dev, "Failed to enable gfx11 ras feature\n");
4751
4752         kfree(gfx_common_if);
4753         return 0;
4754 }
4755
4756 static int gfx_v11_0_late_init(void *handle)
4757 {
4758         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4759         int r;
4760
4761         r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0);
4762         if (r)
4763                 return r;
4764
4765         r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0);
4766         if (r)
4767                 return r;
4768
4769         if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(11, 0, 3)) {
4770                 r = gfx_v11_0_ras_late_init(handle);
4771                 if (r)
4772                         return r;
4773         }
4774
4775         return 0;
4776 }
4777
4778 static bool gfx_v11_0_is_rlc_enabled(struct amdgpu_device *adev)
4779 {
4780         uint32_t rlc_cntl;
4781
4782         /* if RLC is not enabled, do nothing */
4783         rlc_cntl = RREG32_SOC15(GC, 0, regRLC_CNTL);
4784         return (REG_GET_FIELD(rlc_cntl, RLC_CNTL, RLC_ENABLE_F32)) ? true : false;
4785 }
4786
4787 static void gfx_v11_0_set_safe_mode(struct amdgpu_device *adev)
4788 {
4789         uint32_t data;
4790         unsigned i;
4791
4792         data = RLC_SAFE_MODE__CMD_MASK;
4793         data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT);
4794
4795         WREG32_SOC15(GC, 0, regRLC_SAFE_MODE, data);
4796
4797         /* wait for RLC_SAFE_MODE */
4798         for (i = 0; i < adev->usec_timeout; i++) {
4799                 if (!REG_GET_FIELD(RREG32_SOC15(GC, 0, regRLC_SAFE_MODE),
4800                                    RLC_SAFE_MODE, CMD))
4801                         break;
4802                 udelay(1);
4803         }
4804 }
4805
4806 static void gfx_v11_0_unset_safe_mode(struct amdgpu_device *adev)
4807 {
4808         WREG32_SOC15(GC, 0, regRLC_SAFE_MODE, RLC_SAFE_MODE__CMD_MASK);
4809 }
4810
4811 static void gfx_v11_0_update_perf_clk(struct amdgpu_device *adev,
4812                                       bool enable)
4813 {
4814         uint32_t def, data;
4815
4816         if (!(adev->cg_flags & AMD_CG_SUPPORT_GFX_PERF_CLK))
4817                 return;
4818
4819         def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
4820
4821         if (enable)
4822                 data &= ~RLC_CGTT_MGCG_OVERRIDE__PERFMON_CLOCK_STATE_MASK;
4823         else
4824                 data |= RLC_CGTT_MGCG_OVERRIDE__PERFMON_CLOCK_STATE_MASK;
4825
4826         if (def != data)
4827                 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data);
4828 }
4829
4830 static void gfx_v11_0_update_sram_fgcg(struct amdgpu_device *adev,
4831                                        bool enable)
4832 {
4833         uint32_t def, data;
4834
4835         if (!(adev->cg_flags & AMD_CG_SUPPORT_GFX_FGCG))
4836                 return;
4837
4838         def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
4839
4840         if (enable)
4841                 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_FGCG_OVERRIDE_MASK;
4842         else
4843                 data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_FGCG_OVERRIDE_MASK;
4844
4845         if (def != data)
4846                 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data);
4847 }
4848
4849 static void gfx_v11_0_update_repeater_fgcg(struct amdgpu_device *adev,
4850                                            bool enable)
4851 {
4852         uint32_t def, data;
4853
4854         if (!(adev->cg_flags & AMD_CG_SUPPORT_REPEATER_FGCG))
4855                 return;
4856
4857         def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
4858
4859         if (enable)
4860                 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_REPEATER_FGCG_OVERRIDE_MASK;
4861         else
4862                 data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_REPEATER_FGCG_OVERRIDE_MASK;
4863
4864         if (def != data)
4865                 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data);
4866 }
4867
4868 static void gfx_v11_0_update_medium_grain_clock_gating(struct amdgpu_device *adev,
4869                                                        bool enable)
4870 {
4871         uint32_t data, def;
4872
4873         if (!(adev->cg_flags & (AMD_CG_SUPPORT_GFX_MGCG | AMD_CG_SUPPORT_GFX_MGLS)))
4874                 return;
4875
4876         /* It is disabled by HW by default */
4877         if (enable) {
4878                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG) {
4879                         /* 1 - RLC_CGTT_MGCG_OVERRIDE */
4880                         def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
4881
4882                         data &= ~(RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
4883                                   RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK |
4884                                   RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK);
4885
4886                         if (def != data)
4887                                 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data);
4888                 }
4889         } else {
4890                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG) {
4891                         def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
4892
4893                         data |= (RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK |
4894                                  RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
4895                                  RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK);
4896
4897                         if (def != data)
4898                                 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data);
4899                 }
4900         }
4901 }
4902
4903 static void gfx_v11_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev,
4904                                                        bool enable)
4905 {
4906         uint32_t def, data;
4907
4908         if (!(adev->cg_flags &
4909               (AMD_CG_SUPPORT_GFX_CGCG |
4910               AMD_CG_SUPPORT_GFX_CGLS |
4911               AMD_CG_SUPPORT_GFX_3D_CGCG |
4912               AMD_CG_SUPPORT_GFX_3D_CGLS)))
4913                 return;
4914
4915         if (enable) {
4916                 def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
4917
4918                 /* unset CGCG override */
4919                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)
4920                         data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGCG_OVERRIDE_MASK;
4921                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
4922                         data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK;
4923                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG ||
4924                     adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS)
4925                         data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_GFX3D_CG_OVERRIDE_MASK;
4926
4927                 /* update CGCG override bits */
4928                 if (def != data)
4929                         WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data);
4930
4931                 /* enable cgcg FSM(0x0000363F) */
4932                 def = data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL);
4933
4934                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG) {
4935                         data &= ~RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD_MASK;
4936                         data |= (0x36 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
4937                                  RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
4938                 }
4939
4940                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) {
4941                         data &= ~RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY_MASK;
4942                         data |= (0x000F << RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
4943                                  RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;
4944                 }
4945
4946                 if (def != data)
4947                         WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL, data);
4948
4949                 /* Program RLC_CGCG_CGLS_CTRL_3D */
4950                 def = data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D);
4951
4952                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG) {
4953                         data &= ~RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD_MASK;
4954                         data |= (0x36 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
4955                                  RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK;
4956                 }
4957
4958                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS) {
4959                         data &= ~RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY_MASK;
4960                         data |= (0xf << RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
4961                                  RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK;
4962                 }
4963
4964                 if (def != data)
4965                         WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D, data);
4966
4967                 /* set IDLE_POLL_COUNT(0x00900100) */
4968                 def = data = RREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_CNTL);
4969
4970                 data &= ~(CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY_MASK | CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK);
4971                 data |= (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) |
4972                         (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
4973
4974                 if (def != data)
4975                         WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_CNTL, data);
4976
4977                 data = RREG32_SOC15(GC, 0, regCP_INT_CNTL);
4978                 data = REG_SET_FIELD(data, CP_INT_CNTL, CNTX_BUSY_INT_ENABLE, 1);
4979                 data = REG_SET_FIELD(data, CP_INT_CNTL, CNTX_EMPTY_INT_ENABLE, 1);
4980                 data = REG_SET_FIELD(data, CP_INT_CNTL, CMP_BUSY_INT_ENABLE, 1);
4981                 data = REG_SET_FIELD(data, CP_INT_CNTL, GFX_IDLE_INT_ENABLE, 1);
4982                 WREG32_SOC15(GC, 0, regCP_INT_CNTL, data);
4983
4984                 data = RREG32_SOC15(GC, 0, regSDMA0_RLC_CGCG_CTRL);
4985                 data = REG_SET_FIELD(data, SDMA0_RLC_CGCG_CTRL, CGCG_INT_ENABLE, 1);
4986                 WREG32_SOC15(GC, 0, regSDMA0_RLC_CGCG_CTRL, data);
4987
4988                 /* Some ASICs only have one SDMA instance, not need to configure SDMA1 */
4989                 if (adev->sdma.num_instances > 1) {
4990                         data = RREG32_SOC15(GC, 0, regSDMA1_RLC_CGCG_CTRL);
4991                         data = REG_SET_FIELD(data, SDMA1_RLC_CGCG_CTRL, CGCG_INT_ENABLE, 1);
4992                         WREG32_SOC15(GC, 0, regSDMA1_RLC_CGCG_CTRL, data);
4993                 }
4994         } else {
4995                 /* Program RLC_CGCG_CGLS_CTRL */
4996                 def = data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL);
4997
4998                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)
4999                         data &= ~RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
5000
5001                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
5002                         data &= ~RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;
5003
5004                 if (def != data)
5005                         WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL, data);
5006
5007                 /* Program RLC_CGCG_CGLS_CTRL_3D */
5008                 def = data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D);
5009
5010                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG)
5011                         data &= ~RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK;
5012                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS)
5013                         data &= ~RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK;
5014
5015                 if (def != data)
5016                         WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D, data);
5017
5018                 data = RREG32_SOC15(GC, 0, regSDMA0_RLC_CGCG_CTRL);
5019                 data &= ~SDMA0_RLC_CGCG_CTRL__CGCG_INT_ENABLE_MASK;
5020                 WREG32_SOC15(GC, 0, regSDMA0_RLC_CGCG_CTRL, data);
5021
5022                 /* Some ASICs only have one SDMA instance, not need to configure SDMA1 */
5023                 if (adev->sdma.num_instances > 1) {
5024                         data = RREG32_SOC15(GC, 0, regSDMA1_RLC_CGCG_CTRL);
5025                         data &= ~SDMA1_RLC_CGCG_CTRL__CGCG_INT_ENABLE_MASK;
5026                         WREG32_SOC15(GC, 0, regSDMA1_RLC_CGCG_CTRL, data);
5027                 }
5028         }
5029 }
5030
5031 static int gfx_v11_0_update_gfx_clock_gating(struct amdgpu_device *adev,
5032                                             bool enable)
5033 {
5034         amdgpu_gfx_rlc_enter_safe_mode(adev);
5035
5036         gfx_v11_0_update_coarse_grain_clock_gating(adev, enable);
5037
5038         gfx_v11_0_update_medium_grain_clock_gating(adev, enable);
5039
5040         gfx_v11_0_update_repeater_fgcg(adev, enable);
5041
5042         gfx_v11_0_update_sram_fgcg(adev, enable);
5043
5044         gfx_v11_0_update_perf_clk(adev, enable);
5045
5046         if (adev->cg_flags &
5047             (AMD_CG_SUPPORT_GFX_MGCG |
5048              AMD_CG_SUPPORT_GFX_CGLS |
5049              AMD_CG_SUPPORT_GFX_CGCG |
5050              AMD_CG_SUPPORT_GFX_3D_CGCG |
5051              AMD_CG_SUPPORT_GFX_3D_CGLS))
5052                 gfx_v11_0_enable_gui_idle_interrupt(adev, enable);
5053
5054         amdgpu_gfx_rlc_exit_safe_mode(adev);
5055
5056         return 0;
5057 }
5058
5059 static void gfx_v11_0_update_spm_vmid(struct amdgpu_device *adev, unsigned vmid)
5060 {
5061         u32 reg, data;
5062
5063         amdgpu_gfx_off_ctrl(adev, false);
5064
5065         reg = SOC15_REG_OFFSET(GC, 0, regRLC_SPM_MC_CNTL);
5066         if (amdgpu_sriov_is_pp_one_vf(adev))
5067                 data = RREG32_NO_KIQ(reg);
5068         else
5069                 data = RREG32(reg);
5070
5071         data &= ~RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK;
5072         data |= (vmid & RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK) << RLC_SPM_MC_CNTL__RLC_SPM_VMID__SHIFT;
5073
5074         if (amdgpu_sriov_is_pp_one_vf(adev))
5075                 WREG32_SOC15_NO_KIQ(GC, 0, regRLC_SPM_MC_CNTL, data);
5076         else
5077                 WREG32_SOC15(GC, 0, regRLC_SPM_MC_CNTL, data);
5078
5079         amdgpu_gfx_off_ctrl(adev, true);
5080 }
5081
5082 static const struct amdgpu_rlc_funcs gfx_v11_0_rlc_funcs = {
5083         .is_rlc_enabled = gfx_v11_0_is_rlc_enabled,
5084         .set_safe_mode = gfx_v11_0_set_safe_mode,
5085         .unset_safe_mode = gfx_v11_0_unset_safe_mode,
5086         .init = gfx_v11_0_rlc_init,
5087         .get_csb_size = gfx_v11_0_get_csb_size,
5088         .get_csb_buffer = gfx_v11_0_get_csb_buffer,
5089         .resume = gfx_v11_0_rlc_resume,
5090         .stop = gfx_v11_0_rlc_stop,
5091         .reset = gfx_v11_0_rlc_reset,
5092         .start = gfx_v11_0_rlc_start,
5093         .update_spm_vmid = gfx_v11_0_update_spm_vmid,
5094 };
5095
5096 static void gfx_v11_cntl_power_gating(struct amdgpu_device *adev, bool enable)
5097 {
5098         u32 data = RREG32_SOC15(GC, 0, regRLC_PG_CNTL);
5099
5100         if (enable && (adev->pg_flags & AMD_PG_SUPPORT_GFX_PG))
5101                 data |= RLC_PG_CNTL__GFX_POWER_GATING_ENABLE_MASK;
5102         else
5103                 data &= ~RLC_PG_CNTL__GFX_POWER_GATING_ENABLE_MASK;
5104
5105         WREG32_SOC15(GC, 0, regRLC_PG_CNTL, data);
5106
5107         // Program RLC_PG_DELAY3 for CGPG hysteresis
5108         if (enable && (adev->pg_flags & AMD_PG_SUPPORT_GFX_PG)) {
5109                 switch (adev->ip_versions[GC_HWIP][0]) {
5110                 case IP_VERSION(11, 0, 1):
5111                 case IP_VERSION(11, 0, 4):
5112                         WREG32_SOC15(GC, 0, regRLC_PG_DELAY_3, RLC_PG_DELAY_3_DEFAULT_GC_11_0_1);
5113                         break;
5114                 default:
5115                         break;
5116                 }
5117         }
5118 }
5119
5120 static void gfx_v11_cntl_pg(struct amdgpu_device *adev, bool enable)
5121 {
5122         amdgpu_gfx_rlc_enter_safe_mode(adev);
5123
5124         gfx_v11_cntl_power_gating(adev, enable);
5125
5126         amdgpu_gfx_rlc_exit_safe_mode(adev);
5127 }
5128
5129 static int gfx_v11_0_set_powergating_state(void *handle,
5130                                            enum amd_powergating_state state)
5131 {
5132         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5133         bool enable = (state == AMD_PG_STATE_GATE);
5134
5135         if (amdgpu_sriov_vf(adev))
5136                 return 0;
5137
5138         switch (adev->ip_versions[GC_HWIP][0]) {
5139         case IP_VERSION(11, 0, 0):
5140         case IP_VERSION(11, 0, 2):
5141         case IP_VERSION(11, 0, 3):
5142                 amdgpu_gfx_off_ctrl(adev, enable);
5143                 break;
5144         case IP_VERSION(11, 0, 1):
5145         case IP_VERSION(11, 0, 4):
5146                 gfx_v11_cntl_pg(adev, enable);
5147                 amdgpu_gfx_off_ctrl(adev, enable);
5148                 break;
5149         default:
5150                 break;
5151         }
5152
5153         return 0;
5154 }
5155
5156 static int gfx_v11_0_set_clockgating_state(void *handle,
5157                                           enum amd_clockgating_state state)
5158 {
5159         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5160
5161         if (amdgpu_sriov_vf(adev))
5162                 return 0;
5163
5164         switch (adev->ip_versions[GC_HWIP][0]) {
5165         case IP_VERSION(11, 0, 0):
5166         case IP_VERSION(11, 0, 1):
5167         case IP_VERSION(11, 0, 2):
5168         case IP_VERSION(11, 0, 3):
5169         case IP_VERSION(11, 0, 4):
5170                 gfx_v11_0_update_gfx_clock_gating(adev,
5171                                 state ==  AMD_CG_STATE_GATE);
5172                 break;
5173         default:
5174                 break;
5175         }
5176
5177         return 0;
5178 }
5179
5180 static void gfx_v11_0_get_clockgating_state(void *handle, u64 *flags)
5181 {
5182         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5183         int data;
5184
5185         /* AMD_CG_SUPPORT_GFX_MGCG */
5186         data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
5187         if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK))
5188                 *flags |= AMD_CG_SUPPORT_GFX_MGCG;
5189
5190         /* AMD_CG_SUPPORT_REPEATER_FGCG */
5191         if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_REPEATER_FGCG_OVERRIDE_MASK))
5192                 *flags |= AMD_CG_SUPPORT_REPEATER_FGCG;
5193
5194         /* AMD_CG_SUPPORT_GFX_FGCG */
5195         if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_FGCG_OVERRIDE_MASK))
5196                 *flags |= AMD_CG_SUPPORT_GFX_FGCG;
5197
5198         /* AMD_CG_SUPPORT_GFX_PERF_CLK */
5199         if (!(data & RLC_CGTT_MGCG_OVERRIDE__PERFMON_CLOCK_STATE_MASK))
5200                 *flags |= AMD_CG_SUPPORT_GFX_PERF_CLK;
5201
5202         /* AMD_CG_SUPPORT_GFX_CGCG */
5203         data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL);
5204         if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK)
5205                 *flags |= AMD_CG_SUPPORT_GFX_CGCG;
5206
5207         /* AMD_CG_SUPPORT_GFX_CGLS */
5208         if (data & RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK)
5209                 *flags |= AMD_CG_SUPPORT_GFX_CGLS;
5210
5211         /* AMD_CG_SUPPORT_GFX_3D_CGCG */
5212         data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D);
5213         if (data & RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK)
5214                 *flags |= AMD_CG_SUPPORT_GFX_3D_CGCG;
5215
5216         /* AMD_CG_SUPPORT_GFX_3D_CGLS */
5217         if (data & RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK)
5218                 *flags |= AMD_CG_SUPPORT_GFX_3D_CGLS;
5219 }
5220
5221 static u64 gfx_v11_0_ring_get_rptr_gfx(struct amdgpu_ring *ring)
5222 {
5223         /* gfx11 is 32bit rptr*/
5224         return *(uint32_t *)ring->rptr_cpu_addr;
5225 }
5226
5227 static u64 gfx_v11_0_ring_get_wptr_gfx(struct amdgpu_ring *ring)
5228 {
5229         struct amdgpu_device *adev = ring->adev;
5230         u64 wptr;
5231
5232         /* XXX check if swapping is necessary on BE */
5233         if (ring->use_doorbell) {
5234                 wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr);
5235         } else {
5236                 wptr = RREG32_SOC15(GC, 0, regCP_RB0_WPTR);
5237                 wptr += (u64)RREG32_SOC15(GC, 0, regCP_RB0_WPTR_HI) << 32;
5238         }
5239
5240         return wptr;
5241 }
5242
5243 static void gfx_v11_0_ring_set_wptr_gfx(struct amdgpu_ring *ring)
5244 {
5245         struct amdgpu_device *adev = ring->adev;
5246         uint32_t *wptr_saved;
5247         uint32_t *is_queue_unmap;
5248         uint64_t aggregated_db_index;
5249         uint32_t mqd_size = adev->mqds[AMDGPU_HW_IP_GFX].mqd_size;
5250         uint64_t wptr_tmp;
5251
5252         if (ring->is_mes_queue) {
5253                 wptr_saved = (uint32_t *)(ring->mqd_ptr + mqd_size);
5254                 is_queue_unmap = (uint32_t *)(ring->mqd_ptr + mqd_size +
5255                                               sizeof(uint32_t));
5256                 aggregated_db_index =
5257                         amdgpu_mes_get_aggregated_doorbell_index(adev,
5258                                                                  ring->hw_prio);
5259
5260                 wptr_tmp = ring->wptr & ring->buf_mask;
5261                 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, wptr_tmp);
5262                 *wptr_saved = wptr_tmp;
5263                 /* assume doorbell always being used by mes mapped queue */
5264                 if (*is_queue_unmap) {
5265                         WDOORBELL64(aggregated_db_index, wptr_tmp);
5266                         WDOORBELL64(ring->doorbell_index, wptr_tmp);
5267                 } else {
5268                         WDOORBELL64(ring->doorbell_index, wptr_tmp);
5269
5270                         if (*is_queue_unmap)
5271                                 WDOORBELL64(aggregated_db_index, wptr_tmp);
5272                 }
5273         } else {
5274                 if (ring->use_doorbell) {
5275                         /* XXX check if swapping is necessary on BE */
5276                         atomic64_set((atomic64_t *)ring->wptr_cpu_addr,
5277                                      ring->wptr);
5278                         WDOORBELL64(ring->doorbell_index, ring->wptr);
5279                 } else {
5280                         WREG32_SOC15(GC, 0, regCP_RB0_WPTR,
5281                                      lower_32_bits(ring->wptr));
5282                         WREG32_SOC15(GC, 0, regCP_RB0_WPTR_HI,
5283                                      upper_32_bits(ring->wptr));
5284                 }
5285         }
5286 }
5287
5288 static u64 gfx_v11_0_ring_get_rptr_compute(struct amdgpu_ring *ring)
5289 {
5290         /* gfx11 hardware is 32bit rptr */
5291         return *(uint32_t *)ring->rptr_cpu_addr;
5292 }
5293
5294 static u64 gfx_v11_0_ring_get_wptr_compute(struct amdgpu_ring *ring)
5295 {
5296         u64 wptr;
5297
5298         /* XXX check if swapping is necessary on BE */
5299         if (ring->use_doorbell)
5300                 wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr);
5301         else
5302                 BUG();
5303         return wptr;
5304 }
5305
5306 static void gfx_v11_0_ring_set_wptr_compute(struct amdgpu_ring *ring)
5307 {
5308         struct amdgpu_device *adev = ring->adev;
5309         uint32_t *wptr_saved;
5310         uint32_t *is_queue_unmap;
5311         uint64_t aggregated_db_index;
5312         uint32_t mqd_size = adev->mqds[AMDGPU_HW_IP_COMPUTE].mqd_size;
5313         uint64_t wptr_tmp;
5314
5315         if (ring->is_mes_queue) {
5316                 wptr_saved = (uint32_t *)(ring->mqd_ptr + mqd_size);
5317                 is_queue_unmap = (uint32_t *)(ring->mqd_ptr + mqd_size +
5318                                               sizeof(uint32_t));
5319                 aggregated_db_index =
5320                         amdgpu_mes_get_aggregated_doorbell_index(adev,
5321                                                                  ring->hw_prio);
5322
5323                 wptr_tmp = ring->wptr & ring->buf_mask;
5324                 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, wptr_tmp);
5325                 *wptr_saved = wptr_tmp;
5326                 /* assume doorbell always used by mes mapped queue */
5327                 if (*is_queue_unmap) {
5328                         WDOORBELL64(aggregated_db_index, wptr_tmp);
5329                         WDOORBELL64(ring->doorbell_index, wptr_tmp);
5330                 } else {
5331                         WDOORBELL64(ring->doorbell_index, wptr_tmp);
5332
5333                         if (*is_queue_unmap)
5334                                 WDOORBELL64(aggregated_db_index, wptr_tmp);
5335                 }
5336         } else {
5337                 /* XXX check if swapping is necessary on BE */
5338                 if (ring->use_doorbell) {
5339                         atomic64_set((atomic64_t *)ring->wptr_cpu_addr,
5340                                      ring->wptr);
5341                         WDOORBELL64(ring->doorbell_index, ring->wptr);
5342                 } else {
5343                         BUG(); /* only DOORBELL method supported on gfx11 now */
5344                 }
5345         }
5346 }
5347
5348 static void gfx_v11_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
5349 {
5350         struct amdgpu_device *adev = ring->adev;
5351         u32 ref_and_mask, reg_mem_engine;
5352         const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg;
5353
5354         if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
5355                 switch (ring->me) {
5356                 case 1:
5357                         ref_and_mask = nbio_hf_reg->ref_and_mask_cp2 << ring->pipe;
5358                         break;
5359                 case 2:
5360                         ref_and_mask = nbio_hf_reg->ref_and_mask_cp6 << ring->pipe;
5361                         break;
5362                 default:
5363                         return;
5364                 }
5365                 reg_mem_engine = 0;
5366         } else {
5367                 ref_and_mask = nbio_hf_reg->ref_and_mask_cp0;
5368                 reg_mem_engine = 1; /* pfp */
5369         }
5370
5371         gfx_v11_0_wait_reg_mem(ring, reg_mem_engine, 0, 1,
5372                                adev->nbio.funcs->get_hdp_flush_req_offset(adev),
5373                                adev->nbio.funcs->get_hdp_flush_done_offset(adev),
5374                                ref_and_mask, ref_and_mask, 0x20);
5375 }
5376
5377 static void gfx_v11_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
5378                                        struct amdgpu_job *job,
5379                                        struct amdgpu_ib *ib,
5380                                        uint32_t flags)
5381 {
5382         unsigned vmid = AMDGPU_JOB_GET_VMID(job);
5383         u32 header, control = 0;
5384
5385         BUG_ON(ib->flags & AMDGPU_IB_FLAG_CE);
5386
5387         header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
5388
5389         control |= ib->length_dw | (vmid << 24);
5390
5391         if (amdgpu_mcbp && (ib->flags & AMDGPU_IB_FLAG_PREEMPT)) {
5392                 control |= INDIRECT_BUFFER_PRE_ENB(1);
5393
5394                 if (flags & AMDGPU_IB_PREEMPTED)
5395                         control |= INDIRECT_BUFFER_PRE_RESUME(1);
5396
5397                 if (vmid)
5398                         gfx_v11_0_ring_emit_de_meta(ring,
5399                                     (!amdgpu_sriov_vf(ring->adev) && flags & AMDGPU_IB_PREEMPTED) ? true : false);
5400         }
5401
5402         if (ring->is_mes_queue)
5403                 /* inherit vmid from mqd */
5404                 control |= 0x400000;
5405
5406         amdgpu_ring_write(ring, header);
5407         BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
5408         amdgpu_ring_write(ring,
5409 #ifdef __BIG_ENDIAN
5410                 (2 << 0) |
5411 #endif
5412                 lower_32_bits(ib->gpu_addr));
5413         amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
5414         amdgpu_ring_write(ring, control);
5415 }
5416
5417 static void gfx_v11_0_ring_emit_ib_compute(struct amdgpu_ring *ring,
5418                                            struct amdgpu_job *job,
5419                                            struct amdgpu_ib *ib,
5420                                            uint32_t flags)
5421 {
5422         unsigned vmid = AMDGPU_JOB_GET_VMID(job);
5423         u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24);
5424
5425         if (ring->is_mes_queue)
5426                 /* inherit vmid from mqd */
5427                 control |= 0x40000000;
5428
5429         /* Currently, there is a high possibility to get wave ID mismatch
5430          * between ME and GDS, leading to a hw deadlock, because ME generates
5431          * different wave IDs than the GDS expects. This situation happens
5432          * randomly when at least 5 compute pipes use GDS ordered append.
5433          * The wave IDs generated by ME are also wrong after suspend/resume.
5434          * Those are probably bugs somewhere else in the kernel driver.
5435          *
5436          * Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and
5437          * GDS to 0 for this ring (me/pipe).
5438          */
5439         if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) {
5440                 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
5441                 amdgpu_ring_write(ring, regGDS_COMPUTE_MAX_WAVE_ID);
5442                 amdgpu_ring_write(ring, ring->adev->gds.gds_compute_max_wave_id);
5443         }
5444
5445         amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
5446         BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
5447         amdgpu_ring_write(ring,
5448 #ifdef __BIG_ENDIAN
5449                                 (2 << 0) |
5450 #endif
5451                                 lower_32_bits(ib->gpu_addr));
5452         amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
5453         amdgpu_ring_write(ring, control);
5454 }
5455
5456 static void gfx_v11_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
5457                                      u64 seq, unsigned flags)
5458 {
5459         bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
5460         bool int_sel = flags & AMDGPU_FENCE_FLAG_INT;
5461
5462         /* RELEASE_MEM - flush caches, send int */
5463         amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 6));
5464         amdgpu_ring_write(ring, (PACKET3_RELEASE_MEM_GCR_SEQ |
5465                                  PACKET3_RELEASE_MEM_GCR_GL2_WB |
5466                                  PACKET3_RELEASE_MEM_GCR_GL2_INV |
5467                                  PACKET3_RELEASE_MEM_GCR_GL2_US |
5468                                  PACKET3_RELEASE_MEM_GCR_GL1_INV |
5469                                  PACKET3_RELEASE_MEM_GCR_GLV_INV |
5470                                  PACKET3_RELEASE_MEM_GCR_GLM_INV |
5471                                  PACKET3_RELEASE_MEM_GCR_GLM_WB |
5472                                  PACKET3_RELEASE_MEM_CACHE_POLICY(3) |
5473                                  PACKET3_RELEASE_MEM_EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
5474                                  PACKET3_RELEASE_MEM_EVENT_INDEX(5)));
5475         amdgpu_ring_write(ring, (PACKET3_RELEASE_MEM_DATA_SEL(write64bit ? 2 : 1) |
5476                                  PACKET3_RELEASE_MEM_INT_SEL(int_sel ? 2 : 0)));
5477
5478         /*
5479          * the address should be Qword aligned if 64bit write, Dword
5480          * aligned if only send 32bit data low (discard data high)
5481          */
5482         if (write64bit)
5483                 BUG_ON(addr & 0x7);
5484         else
5485                 BUG_ON(addr & 0x3);
5486         amdgpu_ring_write(ring, lower_32_bits(addr));
5487         amdgpu_ring_write(ring, upper_32_bits(addr));
5488         amdgpu_ring_write(ring, lower_32_bits(seq));
5489         amdgpu_ring_write(ring, upper_32_bits(seq));
5490         amdgpu_ring_write(ring, ring->is_mes_queue ?
5491                          (ring->hw_queue_id | AMDGPU_FENCE_MES_QUEUE_FLAG) : 0);
5492 }
5493
5494 static void gfx_v11_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
5495 {
5496         int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
5497         uint32_t seq = ring->fence_drv.sync_seq;
5498         uint64_t addr = ring->fence_drv.gpu_addr;
5499
5500         gfx_v11_0_wait_reg_mem(ring, usepfp, 1, 0, lower_32_bits(addr),
5501                                upper_32_bits(addr), seq, 0xffffffff, 4);
5502 }
5503
5504 static void gfx_v11_0_ring_invalidate_tlbs(struct amdgpu_ring *ring,
5505                                    uint16_t pasid, uint32_t flush_type,
5506                                    bool all_hub, uint8_t dst_sel)
5507 {
5508         amdgpu_ring_write(ring, PACKET3(PACKET3_INVALIDATE_TLBS, 0));
5509         amdgpu_ring_write(ring,
5510                           PACKET3_INVALIDATE_TLBS_DST_SEL(dst_sel) |
5511                           PACKET3_INVALIDATE_TLBS_ALL_HUB(all_hub) |
5512                           PACKET3_INVALIDATE_TLBS_PASID(pasid) |
5513                           PACKET3_INVALIDATE_TLBS_FLUSH_TYPE(flush_type));
5514 }
5515
5516 static void gfx_v11_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
5517                                          unsigned vmid, uint64_t pd_addr)
5518 {
5519         if (ring->is_mes_queue)
5520                 gfx_v11_0_ring_invalidate_tlbs(ring, 0, 0, false, 0);
5521         else
5522                 amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
5523
5524         /* compute doesn't have PFP */
5525         if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) {
5526                 /* sync PFP to ME, otherwise we might get invalid PFP reads */
5527                 amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5528                 amdgpu_ring_write(ring, 0x0);
5529         }
5530 }
5531
5532 static void gfx_v11_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr,
5533                                           u64 seq, unsigned int flags)
5534 {
5535         struct amdgpu_device *adev = ring->adev;
5536
5537         /* we only allocate 32bit for each seq wb address */
5538         BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
5539
5540         /* write fence seq to the "addr" */
5541         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5542         amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5543                                  WRITE_DATA_DST_SEL(5) | WR_CONFIRM));
5544         amdgpu_ring_write(ring, lower_32_bits(addr));
5545         amdgpu_ring_write(ring, upper_32_bits(addr));
5546         amdgpu_ring_write(ring, lower_32_bits(seq));
5547
5548         if (flags & AMDGPU_FENCE_FLAG_INT) {
5549                 /* set register to trigger INT */
5550                 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5551                 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5552                                          WRITE_DATA_DST_SEL(0) | WR_CONFIRM));
5553                 amdgpu_ring_write(ring, SOC15_REG_OFFSET(GC, 0, regCPC_INT_STATUS));
5554                 amdgpu_ring_write(ring, 0);
5555                 amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */
5556         }
5557 }
5558
5559 static void gfx_v11_0_ring_emit_cntxcntl(struct amdgpu_ring *ring,
5560                                          uint32_t flags)
5561 {
5562         uint32_t dw2 = 0;
5563
5564         dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */
5565         if (flags & AMDGPU_HAVE_CTX_SWITCH) {
5566                 /* set load_global_config & load_global_uconfig */
5567                 dw2 |= 0x8001;
5568                 /* set load_cs_sh_regs */
5569                 dw2 |= 0x01000000;
5570                 /* set load_per_context_state & load_gfx_sh_regs for GFX */
5571                 dw2 |= 0x10002;
5572         }
5573
5574         amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
5575         amdgpu_ring_write(ring, dw2);
5576         amdgpu_ring_write(ring, 0);
5577 }
5578
5579 static unsigned gfx_v11_0_ring_emit_init_cond_exec(struct amdgpu_ring *ring)
5580 {
5581         unsigned ret;
5582
5583         amdgpu_ring_write(ring, PACKET3(PACKET3_COND_EXEC, 3));
5584         amdgpu_ring_write(ring, lower_32_bits(ring->cond_exe_gpu_addr));
5585         amdgpu_ring_write(ring, upper_32_bits(ring->cond_exe_gpu_addr));
5586         amdgpu_ring_write(ring, 0); /* discard following DWs if *cond_exec_gpu_addr==0 */
5587         ret = ring->wptr & ring->buf_mask;
5588         amdgpu_ring_write(ring, 0x55aa55aa); /* patch dummy value later */
5589
5590         return ret;
5591 }
5592
5593 static void gfx_v11_0_ring_emit_patch_cond_exec(struct amdgpu_ring *ring, unsigned offset)
5594 {
5595         unsigned cur;
5596         BUG_ON(offset > ring->buf_mask);
5597         BUG_ON(ring->ring[offset] != 0x55aa55aa);
5598
5599         cur = (ring->wptr - 1) & ring->buf_mask;
5600         if (likely(cur > offset))
5601                 ring->ring[offset] = cur - offset;
5602         else
5603                 ring->ring[offset] = (ring->buf_mask + 1) - offset + cur;
5604 }
5605
5606 static int gfx_v11_0_ring_preempt_ib(struct amdgpu_ring *ring)
5607 {
5608         int i, r = 0;
5609         struct amdgpu_device *adev = ring->adev;
5610         struct amdgpu_kiq *kiq = &adev->gfx.kiq;
5611         struct amdgpu_ring *kiq_ring = &kiq->ring;
5612         unsigned long flags;
5613
5614         if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues)
5615                 return -EINVAL;
5616
5617         spin_lock_irqsave(&kiq->ring_lock, flags);
5618
5619         if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size)) {
5620                 spin_unlock_irqrestore(&kiq->ring_lock, flags);
5621                 return -ENOMEM;
5622         }
5623
5624         /* assert preemption condition */
5625         amdgpu_ring_set_preempt_cond_exec(ring, false);
5626
5627         /* assert IB preemption, emit the trailing fence */
5628         kiq->pmf->kiq_unmap_queues(kiq_ring, ring, PREEMPT_QUEUES_NO_UNMAP,
5629                                    ring->trail_fence_gpu_addr,
5630                                    ++ring->trail_seq);
5631         amdgpu_ring_commit(kiq_ring);
5632
5633         spin_unlock_irqrestore(&kiq->ring_lock, flags);
5634
5635         /* poll the trailing fence */
5636         for (i = 0; i < adev->usec_timeout; i++) {
5637                 if (ring->trail_seq ==
5638                     le32_to_cpu(*(ring->trail_fence_cpu_addr)))
5639                         break;
5640                 udelay(1);
5641         }
5642
5643         if (i >= adev->usec_timeout) {
5644                 r = -EINVAL;
5645                 DRM_ERROR("ring %d failed to preempt ib\n", ring->idx);
5646         }
5647
5648         /* deassert preemption condition */
5649         amdgpu_ring_set_preempt_cond_exec(ring, true);
5650         return r;
5651 }
5652
5653 static void gfx_v11_0_ring_emit_de_meta(struct amdgpu_ring *ring, bool resume)
5654 {
5655         struct amdgpu_device *adev = ring->adev;
5656         struct v10_de_ib_state de_payload = {0};
5657         uint64_t offset, gds_addr, de_payload_gpu_addr;
5658         void *de_payload_cpu_addr;
5659         int cnt;
5660
5661         if (ring->is_mes_queue) {
5662                 offset = offsetof(struct amdgpu_mes_ctx_meta_data,
5663                                   gfx[0].gfx_meta_data) +
5664                         offsetof(struct v10_gfx_meta_data, de_payload);
5665                 de_payload_gpu_addr =
5666                         amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset);
5667                 de_payload_cpu_addr =
5668                         amdgpu_mes_ctx_get_offs_cpu_addr(ring, offset);
5669
5670                 offset = offsetof(struct amdgpu_mes_ctx_meta_data,
5671                                   gfx[0].gds_backup) +
5672                         offsetof(struct v10_gfx_meta_data, de_payload);
5673                 gds_addr = amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset);
5674         } else {
5675                 offset = offsetof(struct v10_gfx_meta_data, de_payload);
5676                 de_payload_gpu_addr = amdgpu_csa_vaddr(ring->adev) + offset;
5677                 de_payload_cpu_addr = adev->virt.csa_cpu_addr + offset;
5678
5679                 gds_addr = ALIGN(amdgpu_csa_vaddr(ring->adev) +
5680                                  AMDGPU_CSA_SIZE - adev->gds.gds_size,
5681                                  PAGE_SIZE);
5682         }
5683
5684         de_payload.gds_backup_addrlo = lower_32_bits(gds_addr);
5685         de_payload.gds_backup_addrhi = upper_32_bits(gds_addr);
5686
5687         cnt = (sizeof(de_payload) >> 2) + 4 - 2;
5688         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt));
5689         amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
5690                                  WRITE_DATA_DST_SEL(8) |
5691                                  WR_CONFIRM) |
5692                                  WRITE_DATA_CACHE_POLICY(0));
5693         amdgpu_ring_write(ring, lower_32_bits(de_payload_gpu_addr));
5694         amdgpu_ring_write(ring, upper_32_bits(de_payload_gpu_addr));
5695
5696         if (resume)
5697                 amdgpu_ring_write_multiple(ring, de_payload_cpu_addr,
5698                                            sizeof(de_payload) >> 2);
5699         else
5700                 amdgpu_ring_write_multiple(ring, (void *)&de_payload,
5701                                            sizeof(de_payload) >> 2);
5702 }
5703
5704 static void gfx_v11_0_ring_emit_frame_cntl(struct amdgpu_ring *ring, bool start,
5705                                     bool secure)
5706 {
5707         uint32_t v = secure ? FRAME_TMZ : 0;
5708
5709         amdgpu_ring_write(ring, PACKET3(PACKET3_FRAME_CONTROL, 0));
5710         amdgpu_ring_write(ring, v | FRAME_CMD(start ? 0 : 1));
5711 }
5712
5713 static void gfx_v11_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg,
5714                                      uint32_t reg_val_offs)
5715 {
5716         struct amdgpu_device *adev = ring->adev;
5717
5718         amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));
5719         amdgpu_ring_write(ring, 0 |     /* src: register*/
5720                                 (5 << 8) |      /* dst: memory */
5721                                 (1 << 20));     /* write confirm */
5722         amdgpu_ring_write(ring, reg);
5723         amdgpu_ring_write(ring, 0);
5724         amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +
5725                                 reg_val_offs * 4));
5726         amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +
5727                                 reg_val_offs * 4));
5728 }
5729
5730 static void gfx_v11_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg,
5731                                    uint32_t val)
5732 {
5733         uint32_t cmd = 0;
5734
5735         switch (ring->funcs->type) {
5736         case AMDGPU_RING_TYPE_GFX:
5737                 cmd = WRITE_DATA_ENGINE_SEL(1) | WR_CONFIRM;
5738                 break;
5739         case AMDGPU_RING_TYPE_KIQ:
5740                 cmd = (1 << 16); /* no inc addr */
5741                 break;
5742         default:
5743                 cmd = WR_CONFIRM;
5744                 break;
5745         }
5746         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5747         amdgpu_ring_write(ring, cmd);
5748         amdgpu_ring_write(ring, reg);
5749         amdgpu_ring_write(ring, 0);
5750         amdgpu_ring_write(ring, val);
5751 }
5752
5753 static void gfx_v11_0_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
5754                                         uint32_t val, uint32_t mask)
5755 {
5756         gfx_v11_0_wait_reg_mem(ring, 0, 0, 0, reg, 0, val, mask, 0x20);
5757 }
5758
5759 static void gfx_v11_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring,
5760                                                    uint32_t reg0, uint32_t reg1,
5761                                                    uint32_t ref, uint32_t mask)
5762 {
5763         int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
5764
5765         gfx_v11_0_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1,
5766                                ref, mask, 0x20);
5767 }
5768
5769 static void gfx_v11_0_ring_soft_recovery(struct amdgpu_ring *ring,
5770                                          unsigned vmid)
5771 {
5772         struct amdgpu_device *adev = ring->adev;
5773         uint32_t value = 0;
5774
5775         value = REG_SET_FIELD(value, SQ_CMD, CMD, 0x03);
5776         value = REG_SET_FIELD(value, SQ_CMD, MODE, 0x01);
5777         value = REG_SET_FIELD(value, SQ_CMD, CHECK_VMID, 1);
5778         value = REG_SET_FIELD(value, SQ_CMD, VM_ID, vmid);
5779         WREG32_SOC15(GC, 0, regSQ_CMD, value);
5780 }
5781
5782 static void
5783 gfx_v11_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev,
5784                                       uint32_t me, uint32_t pipe,
5785                                       enum amdgpu_interrupt_state state)
5786 {
5787         uint32_t cp_int_cntl, cp_int_cntl_reg;
5788
5789         if (!me) {
5790                 switch (pipe) {
5791                 case 0:
5792                         cp_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_INT_CNTL_RING0);
5793                         break;
5794                 case 1:
5795                         cp_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_INT_CNTL_RING1);
5796                         break;
5797                 default:
5798                         DRM_DEBUG("invalid pipe %d\n", pipe);
5799                         return;
5800                 }
5801         } else {
5802                 DRM_DEBUG("invalid me %d\n", me);
5803                 return;
5804         }
5805
5806         switch (state) {
5807         case AMDGPU_IRQ_STATE_DISABLE:
5808                 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg);
5809                 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
5810                                             TIME_STAMP_INT_ENABLE, 0);
5811                 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
5812                                             GENERIC0_INT_ENABLE, 0);
5813                 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl);
5814                 break;
5815         case AMDGPU_IRQ_STATE_ENABLE:
5816                 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg);
5817                 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
5818                                             TIME_STAMP_INT_ENABLE, 1);
5819                 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
5820                                             GENERIC0_INT_ENABLE, 1);
5821                 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl);
5822                 break;
5823         default:
5824                 break;
5825         }
5826 }
5827
5828 static void gfx_v11_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev,
5829                                                      int me, int pipe,
5830                                                      enum amdgpu_interrupt_state state)
5831 {
5832         u32 mec_int_cntl, mec_int_cntl_reg;
5833
5834         /*
5835          * amdgpu controls only the first MEC. That's why this function only
5836          * handles the setting of interrupts for this specific MEC. All other
5837          * pipes' interrupts are set by amdkfd.
5838          */
5839
5840         if (me == 1) {
5841                 switch (pipe) {
5842                 case 0:
5843                         mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE0_INT_CNTL);
5844                         break;
5845                 case 1:
5846                         mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE1_INT_CNTL);
5847                         break;
5848                 case 2:
5849                         mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE2_INT_CNTL);
5850                         break;
5851                 case 3:
5852                         mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE3_INT_CNTL);
5853                         break;
5854                 default:
5855                         DRM_DEBUG("invalid pipe %d\n", pipe);
5856                         return;
5857                 }
5858         } else {
5859                 DRM_DEBUG("invalid me %d\n", me);
5860                 return;
5861         }
5862
5863         switch (state) {
5864         case AMDGPU_IRQ_STATE_DISABLE:
5865                 mec_int_cntl = RREG32_SOC15_IP(GC, mec_int_cntl_reg);
5866                 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
5867                                              TIME_STAMP_INT_ENABLE, 0);
5868                 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
5869                                              GENERIC0_INT_ENABLE, 0);
5870                 WREG32_SOC15_IP(GC, mec_int_cntl_reg, mec_int_cntl);
5871                 break;
5872         case AMDGPU_IRQ_STATE_ENABLE:
5873                 mec_int_cntl = RREG32_SOC15_IP(GC, mec_int_cntl_reg);
5874                 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
5875                                              TIME_STAMP_INT_ENABLE, 1);
5876                 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
5877                                              GENERIC0_INT_ENABLE, 1);
5878                 WREG32_SOC15_IP(GC, mec_int_cntl_reg, mec_int_cntl);
5879                 break;
5880         default:
5881                 break;
5882         }
5883 }
5884
5885 #define CP_ME1_PIPE_INST_ADDR_INTERVAL  0x1
5886 #define SET_ECC_ME_PIPE_STATE(reg_addr, state) \
5887         do { \
5888                 uint32_t tmp = RREG32_SOC15_IP(GC, reg_addr); \
5889                 tmp = REG_SET_FIELD(tmp, CP_ME1_PIPE0_INT_CNTL, CP_ECC_ERROR_INT_ENABLE, state); \
5890                 WREG32_SOC15_IP(GC, reg_addr, tmp); \
5891         } while (0)
5892
5893 static int gfx_v11_0_set_cp_ecc_error_state(struct amdgpu_device *adev,
5894                                                         struct amdgpu_irq_src *source,
5895                                                         unsigned type,
5896                                                         enum amdgpu_interrupt_state state)
5897 {
5898         uint32_t ecc_irq_state = 0;
5899         uint32_t pipe0_int_cntl_addr = 0;
5900         int i = 0;
5901
5902         ecc_irq_state = (state == AMDGPU_IRQ_STATE_ENABLE) ? 1 : 0;
5903
5904         pipe0_int_cntl_addr = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE0_INT_CNTL);
5905
5906         WREG32_FIELD15_PREREG(GC, 0, CP_INT_CNTL_RING0, CP_ECC_ERROR_INT_ENABLE, ecc_irq_state);
5907
5908         for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++)
5909                 SET_ECC_ME_PIPE_STATE(pipe0_int_cntl_addr + i * CP_ME1_PIPE_INST_ADDR_INTERVAL,
5910                                         ecc_irq_state);
5911
5912         return 0;
5913 }
5914
5915 static int gfx_v11_0_set_eop_interrupt_state(struct amdgpu_device *adev,
5916                                             struct amdgpu_irq_src *src,
5917                                             unsigned type,
5918                                             enum amdgpu_interrupt_state state)
5919 {
5920         switch (type) {
5921         case AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP:
5922                 gfx_v11_0_set_gfx_eop_interrupt_state(adev, 0, 0, state);
5923                 break;
5924         case AMDGPU_CP_IRQ_GFX_ME0_PIPE1_EOP:
5925                 gfx_v11_0_set_gfx_eop_interrupt_state(adev, 0, 1, state);
5926                 break;
5927         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP:
5928                 gfx_v11_0_set_compute_eop_interrupt_state(adev, 1, 0, state);
5929                 break;
5930         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP:
5931                 gfx_v11_0_set_compute_eop_interrupt_state(adev, 1, 1, state);
5932                 break;
5933         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP:
5934                 gfx_v11_0_set_compute_eop_interrupt_state(adev, 1, 2, state);
5935                 break;
5936         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP:
5937                 gfx_v11_0_set_compute_eop_interrupt_state(adev, 1, 3, state);
5938                 break;
5939         default:
5940                 break;
5941         }
5942         return 0;
5943 }
5944
5945 static int gfx_v11_0_eop_irq(struct amdgpu_device *adev,
5946                              struct amdgpu_irq_src *source,
5947                              struct amdgpu_iv_entry *entry)
5948 {
5949         int i;
5950         u8 me_id, pipe_id, queue_id;
5951         struct amdgpu_ring *ring;
5952         uint32_t mes_queue_id = entry->src_data[0];
5953
5954         DRM_DEBUG("IH: CP EOP\n");
5955
5956         if (adev->enable_mes && (mes_queue_id & AMDGPU_FENCE_MES_QUEUE_FLAG)) {
5957                 struct amdgpu_mes_queue *queue;
5958
5959                 mes_queue_id &= AMDGPU_FENCE_MES_QUEUE_ID_MASK;
5960
5961                 spin_lock(&adev->mes.queue_id_lock);
5962                 queue = idr_find(&adev->mes.queue_id_idr, mes_queue_id);
5963                 if (queue) {
5964                         DRM_DEBUG("process mes queue id = %d\n", mes_queue_id);
5965                         amdgpu_fence_process(queue->ring);
5966                 }
5967                 spin_unlock(&adev->mes.queue_id_lock);
5968         } else {
5969                 me_id = (entry->ring_id & 0x0c) >> 2;
5970                 pipe_id = (entry->ring_id & 0x03) >> 0;
5971                 queue_id = (entry->ring_id & 0x70) >> 4;
5972
5973                 switch (me_id) {
5974                 case 0:
5975                         if (pipe_id == 0)
5976                                 amdgpu_fence_process(&adev->gfx.gfx_ring[0]);
5977                         else
5978                                 amdgpu_fence_process(&adev->gfx.gfx_ring[1]);
5979                         break;
5980                 case 1:
5981                 case 2:
5982                         for (i = 0; i < adev->gfx.num_compute_rings; i++) {
5983                                 ring = &adev->gfx.compute_ring[i];
5984                                 /* Per-queue interrupt is supported for MEC starting from VI.
5985                                  * The interrupt can only be enabled/disabled per pipe instead
5986                                  * of per queue.
5987                                  */
5988                                 if ((ring->me == me_id) &&
5989                                     (ring->pipe == pipe_id) &&
5990                                     (ring->queue == queue_id))
5991                                         amdgpu_fence_process(ring);
5992                         }
5993                         break;
5994                 }
5995         }
5996
5997         return 0;
5998 }
5999
6000 static int gfx_v11_0_set_priv_reg_fault_state(struct amdgpu_device *adev,
6001                                               struct amdgpu_irq_src *source,
6002                                               unsigned type,
6003                                               enum amdgpu_interrupt_state state)
6004 {
6005         switch (state) {
6006         case AMDGPU_IRQ_STATE_DISABLE:
6007         case AMDGPU_IRQ_STATE_ENABLE:
6008                 WREG32_FIELD15_PREREG(GC, 0, CP_INT_CNTL_RING0,
6009                                PRIV_REG_INT_ENABLE,
6010                                state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
6011                 break;
6012         default:
6013                 break;
6014         }
6015
6016         return 0;
6017 }
6018
6019 static int gfx_v11_0_set_priv_inst_fault_state(struct amdgpu_device *adev,
6020                                                struct amdgpu_irq_src *source,
6021                                                unsigned type,
6022                                                enum amdgpu_interrupt_state state)
6023 {
6024         switch (state) {
6025         case AMDGPU_IRQ_STATE_DISABLE:
6026         case AMDGPU_IRQ_STATE_ENABLE:
6027                 WREG32_FIELD15_PREREG(GC, 0, CP_INT_CNTL_RING0,
6028                                PRIV_INSTR_INT_ENABLE,
6029                                state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
6030                 break;
6031         default:
6032                 break;
6033         }
6034
6035         return 0;
6036 }
6037
6038 static void gfx_v11_0_handle_priv_fault(struct amdgpu_device *adev,
6039                                         struct amdgpu_iv_entry *entry)
6040 {
6041         u8 me_id, pipe_id, queue_id;
6042         struct amdgpu_ring *ring;
6043         int i;
6044
6045         me_id = (entry->ring_id & 0x0c) >> 2;
6046         pipe_id = (entry->ring_id & 0x03) >> 0;
6047         queue_id = (entry->ring_id & 0x70) >> 4;
6048
6049         switch (me_id) {
6050         case 0:
6051                 for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
6052                         ring = &adev->gfx.gfx_ring[i];
6053                         /* we only enabled 1 gfx queue per pipe for now */
6054                         if (ring->me == me_id && ring->pipe == pipe_id)
6055                                 drm_sched_fault(&ring->sched);
6056                 }
6057                 break;
6058         case 1:
6059         case 2:
6060                 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
6061                         ring = &adev->gfx.compute_ring[i];
6062                         if (ring->me == me_id && ring->pipe == pipe_id &&
6063                             ring->queue == queue_id)
6064                                 drm_sched_fault(&ring->sched);
6065                 }
6066                 break;
6067         default:
6068                 BUG();
6069                 break;
6070         }
6071 }
6072
6073 static int gfx_v11_0_priv_reg_irq(struct amdgpu_device *adev,
6074                                   struct amdgpu_irq_src *source,
6075                                   struct amdgpu_iv_entry *entry)
6076 {
6077         DRM_ERROR("Illegal register access in command stream\n");
6078         gfx_v11_0_handle_priv_fault(adev, entry);
6079         return 0;
6080 }
6081
6082 static int gfx_v11_0_priv_inst_irq(struct amdgpu_device *adev,
6083                                    struct amdgpu_irq_src *source,
6084                                    struct amdgpu_iv_entry *entry)
6085 {
6086         DRM_ERROR("Illegal instruction in command stream\n");
6087         gfx_v11_0_handle_priv_fault(adev, entry);
6088         return 0;
6089 }
6090
6091 static int gfx_v11_0_rlc_gc_fed_irq(struct amdgpu_device *adev,
6092                                   struct amdgpu_irq_src *source,
6093                                   struct amdgpu_iv_entry *entry)
6094 {
6095         if (adev->gfx.ras && adev->gfx.ras->rlc_gc_fed_irq)
6096                 return adev->gfx.ras->rlc_gc_fed_irq(adev, source, entry);
6097
6098         return 0;
6099 }
6100
6101 #if 0
6102 static int gfx_v11_0_kiq_set_interrupt_state(struct amdgpu_device *adev,
6103                                              struct amdgpu_irq_src *src,
6104                                              unsigned int type,
6105                                              enum amdgpu_interrupt_state state)
6106 {
6107         uint32_t tmp, target;
6108         struct amdgpu_ring *ring = &(adev->gfx.kiq.ring);
6109
6110         target = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE0_INT_CNTL);
6111         target += ring->pipe;
6112
6113         switch (type) {
6114         case AMDGPU_CP_KIQ_IRQ_DRIVER0:
6115                 if (state == AMDGPU_IRQ_STATE_DISABLE) {
6116                         tmp = RREG32_SOC15(GC, 0, regCPC_INT_CNTL);
6117                         tmp = REG_SET_FIELD(tmp, CPC_INT_CNTL,
6118                                             GENERIC2_INT_ENABLE, 0);
6119                         WREG32_SOC15(GC, 0, regCPC_INT_CNTL, tmp);
6120
6121                         tmp = RREG32_SOC15_IP(GC, target);
6122                         tmp = REG_SET_FIELD(tmp, CP_ME1_PIPE0_INT_CNTL,
6123                                             GENERIC2_INT_ENABLE, 0);
6124                         WREG32_SOC15_IP(GC, target, tmp);
6125                 } else {
6126                         tmp = RREG32_SOC15(GC, 0, regCPC_INT_CNTL);
6127                         tmp = REG_SET_FIELD(tmp, CPC_INT_CNTL,
6128                                             GENERIC2_INT_ENABLE, 1);
6129                         WREG32_SOC15(GC, 0, regCPC_INT_CNTL, tmp);
6130
6131                         tmp = RREG32_SOC15_IP(GC, target);
6132                         tmp = REG_SET_FIELD(tmp, CP_ME1_PIPE0_INT_CNTL,
6133                                             GENERIC2_INT_ENABLE, 1);
6134                         WREG32_SOC15_IP(GC, target, tmp);
6135                 }
6136                 break;
6137         default:
6138                 BUG(); /* kiq only support GENERIC2_INT now */
6139                 break;
6140         }
6141         return 0;
6142 }
6143 #endif
6144
6145 static void gfx_v11_0_emit_mem_sync(struct amdgpu_ring *ring)
6146 {
6147         const unsigned int gcr_cntl =
6148                         PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_INV(1) |
6149                         PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_WB(1) |
6150                         PACKET3_ACQUIRE_MEM_GCR_CNTL_GLM_INV(1) |
6151                         PACKET3_ACQUIRE_MEM_GCR_CNTL_GLM_WB(1) |
6152                         PACKET3_ACQUIRE_MEM_GCR_CNTL_GL1_INV(1) |
6153                         PACKET3_ACQUIRE_MEM_GCR_CNTL_GLV_INV(1) |
6154                         PACKET3_ACQUIRE_MEM_GCR_CNTL_GLK_INV(1) |
6155                         PACKET3_ACQUIRE_MEM_GCR_CNTL_GLI_INV(1);
6156
6157         /* ACQUIRE_MEM - make one or more surfaces valid for use by the subsequent operations */
6158         amdgpu_ring_write(ring, PACKET3(PACKET3_ACQUIRE_MEM, 6));
6159         amdgpu_ring_write(ring, 0); /* CP_COHER_CNTL */
6160         amdgpu_ring_write(ring, 0xffffffff);  /* CP_COHER_SIZE */
6161         amdgpu_ring_write(ring, 0xffffff);  /* CP_COHER_SIZE_HI */
6162         amdgpu_ring_write(ring, 0); /* CP_COHER_BASE */
6163         amdgpu_ring_write(ring, 0);  /* CP_COHER_BASE_HI */
6164         amdgpu_ring_write(ring, 0x0000000A); /* POLL_INTERVAL */
6165         amdgpu_ring_write(ring, gcr_cntl); /* GCR_CNTL */
6166 }
6167
6168 static const struct amd_ip_funcs gfx_v11_0_ip_funcs = {
6169         .name = "gfx_v11_0",
6170         .early_init = gfx_v11_0_early_init,
6171         .late_init = gfx_v11_0_late_init,
6172         .sw_init = gfx_v11_0_sw_init,
6173         .sw_fini = gfx_v11_0_sw_fini,
6174         .hw_init = gfx_v11_0_hw_init,
6175         .hw_fini = gfx_v11_0_hw_fini,
6176         .suspend = gfx_v11_0_suspend,
6177         .resume = gfx_v11_0_resume,
6178         .is_idle = gfx_v11_0_is_idle,
6179         .wait_for_idle = gfx_v11_0_wait_for_idle,
6180         .soft_reset = gfx_v11_0_soft_reset,
6181         .check_soft_reset = gfx_v11_0_check_soft_reset,
6182         .post_soft_reset = gfx_v11_0_post_soft_reset,
6183         .set_clockgating_state = gfx_v11_0_set_clockgating_state,
6184         .set_powergating_state = gfx_v11_0_set_powergating_state,
6185         .get_clockgating_state = gfx_v11_0_get_clockgating_state,
6186 };
6187
6188 static const struct amdgpu_ring_funcs gfx_v11_0_ring_funcs_gfx = {
6189         .type = AMDGPU_RING_TYPE_GFX,
6190         .align_mask = 0xff,
6191         .nop = PACKET3(PACKET3_NOP, 0x3FFF),
6192         .support_64bit_ptrs = true,
6193         .secure_submission_supported = true,
6194         .vmhub = AMDGPU_GFXHUB_0,
6195         .get_rptr = gfx_v11_0_ring_get_rptr_gfx,
6196         .get_wptr = gfx_v11_0_ring_get_wptr_gfx,
6197         .set_wptr = gfx_v11_0_ring_set_wptr_gfx,
6198         .emit_frame_size = /* totally 242 maximum if 16 IBs */
6199                 5 + /* COND_EXEC */
6200                 7 + /* PIPELINE_SYNC */
6201                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
6202                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
6203                 2 + /* VM_FLUSH */
6204                 8 + /* FENCE for VM_FLUSH */
6205                 20 + /* GDS switch */
6206                 5 + /* COND_EXEC */
6207                 7 + /* HDP_flush */
6208                 4 + /* VGT_flush */
6209                 31 + /* DE_META */
6210                 3 + /* CNTX_CTRL */
6211                 5 + /* HDP_INVL */
6212                 8 + 8 + /* FENCE x2 */
6213                 8, /* gfx_v11_0_emit_mem_sync */
6214         .emit_ib_size = 4, /* gfx_v11_0_ring_emit_ib_gfx */
6215         .emit_ib = gfx_v11_0_ring_emit_ib_gfx,
6216         .emit_fence = gfx_v11_0_ring_emit_fence,
6217         .emit_pipeline_sync = gfx_v11_0_ring_emit_pipeline_sync,
6218         .emit_vm_flush = gfx_v11_0_ring_emit_vm_flush,
6219         .emit_gds_switch = gfx_v11_0_ring_emit_gds_switch,
6220         .emit_hdp_flush = gfx_v11_0_ring_emit_hdp_flush,
6221         .test_ring = gfx_v11_0_ring_test_ring,
6222         .test_ib = gfx_v11_0_ring_test_ib,
6223         .insert_nop = amdgpu_ring_insert_nop,
6224         .pad_ib = amdgpu_ring_generic_pad_ib,
6225         .emit_cntxcntl = gfx_v11_0_ring_emit_cntxcntl,
6226         .init_cond_exec = gfx_v11_0_ring_emit_init_cond_exec,
6227         .patch_cond_exec = gfx_v11_0_ring_emit_patch_cond_exec,
6228         .preempt_ib = gfx_v11_0_ring_preempt_ib,
6229         .emit_frame_cntl = gfx_v11_0_ring_emit_frame_cntl,
6230         .emit_wreg = gfx_v11_0_ring_emit_wreg,
6231         .emit_reg_wait = gfx_v11_0_ring_emit_reg_wait,
6232         .emit_reg_write_reg_wait = gfx_v11_0_ring_emit_reg_write_reg_wait,
6233         .soft_recovery = gfx_v11_0_ring_soft_recovery,
6234         .emit_mem_sync = gfx_v11_0_emit_mem_sync,
6235 };
6236
6237 static const struct amdgpu_ring_funcs gfx_v11_0_ring_funcs_compute = {
6238         .type = AMDGPU_RING_TYPE_COMPUTE,
6239         .align_mask = 0xff,
6240         .nop = PACKET3(PACKET3_NOP, 0x3FFF),
6241         .support_64bit_ptrs = true,
6242         .vmhub = AMDGPU_GFXHUB_0,
6243         .get_rptr = gfx_v11_0_ring_get_rptr_compute,
6244         .get_wptr = gfx_v11_0_ring_get_wptr_compute,
6245         .set_wptr = gfx_v11_0_ring_set_wptr_compute,
6246         .emit_frame_size =
6247                 20 + /* gfx_v11_0_ring_emit_gds_switch */
6248                 7 + /* gfx_v11_0_ring_emit_hdp_flush */
6249                 5 + /* hdp invalidate */
6250                 7 + /* gfx_v11_0_ring_emit_pipeline_sync */
6251                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
6252                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
6253                 2 + /* gfx_v11_0_ring_emit_vm_flush */
6254                 8 + 8 + 8 + /* gfx_v11_0_ring_emit_fence x3 for user fence, vm fence */
6255                 8, /* gfx_v11_0_emit_mem_sync */
6256         .emit_ib_size = 7, /* gfx_v11_0_ring_emit_ib_compute */
6257         .emit_ib = gfx_v11_0_ring_emit_ib_compute,
6258         .emit_fence = gfx_v11_0_ring_emit_fence,
6259         .emit_pipeline_sync = gfx_v11_0_ring_emit_pipeline_sync,
6260         .emit_vm_flush = gfx_v11_0_ring_emit_vm_flush,
6261         .emit_gds_switch = gfx_v11_0_ring_emit_gds_switch,
6262         .emit_hdp_flush = gfx_v11_0_ring_emit_hdp_flush,
6263         .test_ring = gfx_v11_0_ring_test_ring,
6264         .test_ib = gfx_v11_0_ring_test_ib,
6265         .insert_nop = amdgpu_ring_insert_nop,
6266         .pad_ib = amdgpu_ring_generic_pad_ib,
6267         .emit_wreg = gfx_v11_0_ring_emit_wreg,
6268         .emit_reg_wait = gfx_v11_0_ring_emit_reg_wait,
6269         .emit_reg_write_reg_wait = gfx_v11_0_ring_emit_reg_write_reg_wait,
6270         .emit_mem_sync = gfx_v11_0_emit_mem_sync,
6271 };
6272
6273 static const struct amdgpu_ring_funcs gfx_v11_0_ring_funcs_kiq = {
6274         .type = AMDGPU_RING_TYPE_KIQ,
6275         .align_mask = 0xff,
6276         .nop = PACKET3(PACKET3_NOP, 0x3FFF),
6277         .support_64bit_ptrs = true,
6278         .vmhub = AMDGPU_GFXHUB_0,
6279         .get_rptr = gfx_v11_0_ring_get_rptr_compute,
6280         .get_wptr = gfx_v11_0_ring_get_wptr_compute,
6281         .set_wptr = gfx_v11_0_ring_set_wptr_compute,
6282         .emit_frame_size =
6283                 20 + /* gfx_v11_0_ring_emit_gds_switch */
6284                 7 + /* gfx_v11_0_ring_emit_hdp_flush */
6285                 5 + /*hdp invalidate */
6286                 7 + /* gfx_v11_0_ring_emit_pipeline_sync */
6287                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
6288                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
6289                 2 + /* gfx_v11_0_ring_emit_vm_flush */
6290                 8 + 8 + 8, /* gfx_v11_0_ring_emit_fence_kiq x3 for user fence, vm fence */
6291         .emit_ib_size = 7, /* gfx_v11_0_ring_emit_ib_compute */
6292         .emit_ib = gfx_v11_0_ring_emit_ib_compute,
6293         .emit_fence = gfx_v11_0_ring_emit_fence_kiq,
6294         .test_ring = gfx_v11_0_ring_test_ring,
6295         .test_ib = gfx_v11_0_ring_test_ib,
6296         .insert_nop = amdgpu_ring_insert_nop,
6297         .pad_ib = amdgpu_ring_generic_pad_ib,
6298         .emit_rreg = gfx_v11_0_ring_emit_rreg,
6299         .emit_wreg = gfx_v11_0_ring_emit_wreg,
6300         .emit_reg_wait = gfx_v11_0_ring_emit_reg_wait,
6301         .emit_reg_write_reg_wait = gfx_v11_0_ring_emit_reg_write_reg_wait,
6302 };
6303
6304 static void gfx_v11_0_set_ring_funcs(struct amdgpu_device *adev)
6305 {
6306         int i;
6307
6308         adev->gfx.kiq.ring.funcs = &gfx_v11_0_ring_funcs_kiq;
6309
6310         for (i = 0; i < adev->gfx.num_gfx_rings; i++)
6311                 adev->gfx.gfx_ring[i].funcs = &gfx_v11_0_ring_funcs_gfx;
6312
6313         for (i = 0; i < adev->gfx.num_compute_rings; i++)
6314                 adev->gfx.compute_ring[i].funcs = &gfx_v11_0_ring_funcs_compute;
6315 }
6316
6317 static const struct amdgpu_irq_src_funcs gfx_v11_0_eop_irq_funcs = {
6318         .set = gfx_v11_0_set_eop_interrupt_state,
6319         .process = gfx_v11_0_eop_irq,
6320 };
6321
6322 static const struct amdgpu_irq_src_funcs gfx_v11_0_priv_reg_irq_funcs = {
6323         .set = gfx_v11_0_set_priv_reg_fault_state,
6324         .process = gfx_v11_0_priv_reg_irq,
6325 };
6326
6327 static const struct amdgpu_irq_src_funcs gfx_v11_0_priv_inst_irq_funcs = {
6328         .set = gfx_v11_0_set_priv_inst_fault_state,
6329         .process = gfx_v11_0_priv_inst_irq,
6330 };
6331
6332 static const struct amdgpu_irq_src_funcs gfx_v11_0_cp_ecc_error_irq_funcs = {
6333         .set = gfx_v11_0_set_cp_ecc_error_state,
6334         .process = amdgpu_gfx_cp_ecc_error_irq,
6335 };
6336
6337 static const struct amdgpu_irq_src_funcs gfx_v11_0_rlc_gc_fed_irq_funcs = {
6338         .process = gfx_v11_0_rlc_gc_fed_irq,
6339 };
6340
6341 static void gfx_v11_0_set_irq_funcs(struct amdgpu_device *adev)
6342 {
6343         adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST;
6344         adev->gfx.eop_irq.funcs = &gfx_v11_0_eop_irq_funcs;
6345
6346         adev->gfx.priv_reg_irq.num_types = 1;
6347         adev->gfx.priv_reg_irq.funcs = &gfx_v11_0_priv_reg_irq_funcs;
6348
6349         adev->gfx.priv_inst_irq.num_types = 1;
6350         adev->gfx.priv_inst_irq.funcs = &gfx_v11_0_priv_inst_irq_funcs;
6351
6352         adev->gfx.cp_ecc_error_irq.num_types = 1; /* CP ECC error */
6353         adev->gfx.cp_ecc_error_irq.funcs = &gfx_v11_0_cp_ecc_error_irq_funcs;
6354
6355         adev->gfx.rlc_gc_fed_irq.num_types = 1; /* 0x80 FED error */
6356         adev->gfx.rlc_gc_fed_irq.funcs = &gfx_v11_0_rlc_gc_fed_irq_funcs;
6357
6358 }
6359
6360 static void gfx_v11_0_set_imu_funcs(struct amdgpu_device *adev)
6361 {
6362         if (adev->flags & AMD_IS_APU)
6363                 adev->gfx.imu.mode = MISSION_MODE;
6364         else
6365                 adev->gfx.imu.mode = DEBUG_MODE;
6366
6367         adev->gfx.imu.funcs = &gfx_v11_0_imu_funcs;
6368 }
6369
6370 static void gfx_v11_0_set_rlc_funcs(struct amdgpu_device *adev)
6371 {
6372         adev->gfx.rlc.funcs = &gfx_v11_0_rlc_funcs;
6373 }
6374
6375 static void gfx_v11_0_set_gds_init(struct amdgpu_device *adev)
6376 {
6377         unsigned total_cu = adev->gfx.config.max_cu_per_sh *
6378                             adev->gfx.config.max_sh_per_se *
6379                             adev->gfx.config.max_shader_engines;
6380
6381         adev->gds.gds_size = 0x1000;
6382         adev->gds.gds_compute_max_wave_id = total_cu * 32 - 1;
6383         adev->gds.gws_size = 64;
6384         adev->gds.oa_size = 16;
6385 }
6386
6387 static void gfx_v11_0_set_mqd_funcs(struct amdgpu_device *adev)
6388 {
6389         /* set gfx eng mqd */
6390         adev->mqds[AMDGPU_HW_IP_GFX].mqd_size =
6391                 sizeof(struct v11_gfx_mqd);
6392         adev->mqds[AMDGPU_HW_IP_GFX].init_mqd =
6393                 gfx_v11_0_gfx_mqd_init;
6394         /* set compute eng mqd */
6395         adev->mqds[AMDGPU_HW_IP_COMPUTE].mqd_size =
6396                 sizeof(struct v11_compute_mqd);
6397         adev->mqds[AMDGPU_HW_IP_COMPUTE].init_mqd =
6398                 gfx_v11_0_compute_mqd_init;
6399 }
6400
6401 static void gfx_v11_0_set_user_wgp_inactive_bitmap_per_sh(struct amdgpu_device *adev,
6402                                                           u32 bitmap)
6403 {
6404         u32 data;
6405
6406         if (!bitmap)
6407                 return;
6408
6409         data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_WGPS__SHIFT;
6410         data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_WGPS_MASK;
6411
6412         WREG32_SOC15(GC, 0, regGC_USER_SHADER_ARRAY_CONFIG, data);
6413 }
6414
6415 static u32 gfx_v11_0_get_wgp_active_bitmap_per_sh(struct amdgpu_device *adev)
6416 {
6417         u32 data, wgp_bitmask;
6418         data = RREG32_SOC15(GC, 0, regCC_GC_SHADER_ARRAY_CONFIG);
6419         data |= RREG32_SOC15(GC, 0, regGC_USER_SHADER_ARRAY_CONFIG);
6420
6421         data &= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_WGPS_MASK;
6422         data >>= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_WGPS__SHIFT;
6423
6424         wgp_bitmask =
6425                 amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh >> 1);
6426
6427         return (~data) & wgp_bitmask;
6428 }
6429
6430 static u32 gfx_v11_0_get_cu_active_bitmap_per_sh(struct amdgpu_device *adev)
6431 {
6432         u32 wgp_idx, wgp_active_bitmap;
6433         u32 cu_bitmap_per_wgp, cu_active_bitmap;
6434
6435         wgp_active_bitmap = gfx_v11_0_get_wgp_active_bitmap_per_sh(adev);
6436         cu_active_bitmap = 0;
6437
6438         for (wgp_idx = 0; wgp_idx < 16; wgp_idx++) {
6439                 /* if there is one WGP enabled, it means 2 CUs will be enabled */
6440                 cu_bitmap_per_wgp = 3 << (2 * wgp_idx);
6441                 if (wgp_active_bitmap & (1 << wgp_idx))
6442                         cu_active_bitmap |= cu_bitmap_per_wgp;
6443         }
6444
6445         return cu_active_bitmap;
6446 }
6447
6448 static int gfx_v11_0_get_cu_info(struct amdgpu_device *adev,
6449                                  struct amdgpu_cu_info *cu_info)
6450 {
6451         int i, j, k, counter, active_cu_number = 0;
6452         u32 mask, bitmap;
6453         unsigned disable_masks[8 * 2];
6454
6455         if (!adev || !cu_info)
6456                 return -EINVAL;
6457
6458         amdgpu_gfx_parse_disable_cu(disable_masks, 8, 2);
6459
6460         mutex_lock(&adev->grbm_idx_mutex);
6461         for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
6462                 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
6463                         mask = 1;
6464                         counter = 0;
6465                         gfx_v11_0_select_se_sh(adev, i, j, 0xffffffff);
6466                         if (i < 8 && j < 2)
6467                                 gfx_v11_0_set_user_wgp_inactive_bitmap_per_sh(
6468                                         adev, disable_masks[i * 2 + j]);
6469                         bitmap = gfx_v11_0_get_cu_active_bitmap_per_sh(adev);
6470
6471                         /**
6472                          * GFX11 could support more than 4 SEs, while the bitmap
6473                          * in cu_info struct is 4x4 and ioctl interface struct
6474                          * drm_amdgpu_info_device should keep stable.
6475                          * So we use last two columns of bitmap to store cu mask for
6476                          * SEs 4 to 7, the layout of the bitmap is as below:
6477                          *    SE0: {SH0,SH1} --> {bitmap[0][0], bitmap[0][1]}
6478                          *    SE1: {SH0,SH1} --> {bitmap[1][0], bitmap[1][1]}
6479                          *    SE2: {SH0,SH1} --> {bitmap[2][0], bitmap[2][1]}
6480                          *    SE3: {SH0,SH1} --> {bitmap[3][0], bitmap[3][1]}
6481                          *    SE4: {SH0,SH1} --> {bitmap[0][2], bitmap[0][3]}
6482                          *    SE5: {SH0,SH1} --> {bitmap[1][2], bitmap[1][3]}
6483                          *    SE6: {SH0,SH1} --> {bitmap[2][2], bitmap[2][3]}
6484                          *    SE7: {SH0,SH1} --> {bitmap[3][2], bitmap[3][3]}
6485                          */
6486                         cu_info->bitmap[i % 4][j + (i / 4) * 2] = bitmap;
6487
6488                         for (k = 0; k < adev->gfx.config.max_cu_per_sh; k++) {
6489                                 if (bitmap & mask)
6490                                         counter++;
6491
6492                                 mask <<= 1;
6493                         }
6494                         active_cu_number += counter;
6495                 }
6496         }
6497         gfx_v11_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
6498         mutex_unlock(&adev->grbm_idx_mutex);
6499
6500         cu_info->number = active_cu_number;
6501         cu_info->simd_per_cu = NUM_SIMD_PER_CU;
6502
6503         return 0;
6504 }
6505
6506 const struct amdgpu_ip_block_version gfx_v11_0_ip_block =
6507 {
6508         .type = AMD_IP_BLOCK_TYPE_GFX,
6509         .major = 11,
6510         .minor = 0,
6511         .rev = 0,
6512         .funcs = &gfx_v11_0_ip_funcs,
6513 };