drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c

   1 /*
   2  * Copyright 2021 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23 #include <linux/delay.h>
  24 #include <linux/kernel.h>
  25 #include <linux/firmware.h>
  26 #include <linux/module.h>
  27 #include <linux/pci.h>
  28 #include "amdgpu.h"
  29 #include "amdgpu_gfx.h"
  30 #include "amdgpu_psp.h"
  31 #include "amdgpu_smu.h"
  32 #include "amdgpu_atomfirmware.h"
  33 #include "imu_v11_0.h"
  34 #include "soc21.h"
  35 #include "nvd.h"
  36
  37 #include "gc/gc_11_0_0_offset.h"
  38 #include "gc/gc_11_0_0_sh_mask.h"
  39 #include "smuio/smuio_13_0_6_offset.h"
  40 #include "smuio/smuio_13_0_6_sh_mask.h"
  41 #include "navi10_enum.h"
  42 #include "ivsrcid/gfx/irqsrcs_gfx_11_0_0.h"
  43
  44 #include "soc15.h"
  45 #include "soc15d.h"
  46 #include "clearstate_gfx11.h"
  47 #include "v11_structs.h"
  48 #include "gfx_v11_0.h"
  49 #include "nbio_v4_3.h"
  50 #include "mes_v11_0.h"
  51
  52 #define GFX11_NUM_GFX_RINGS             1
  53 #define GFX11_MEC_HPD_SIZE      2048
  54
  55 #define RLCG_UCODE_LOADING_START_ADDRESS        0x00002000L
  56
  57 #define regCGTT_WD_CLK_CTRL             0x5086
  58 #define regCGTT_WD_CLK_CTRL_BASE_IDX    1
  59
  60 MODULE_FIRMWARE("amdgpu/gc_11_0_0_pfp.bin");
  61 MODULE_FIRMWARE("amdgpu/gc_11_0_0_me.bin");
  62 MODULE_FIRMWARE("amdgpu/gc_11_0_0_mec.bin");
  63 MODULE_FIRMWARE("amdgpu/gc_11_0_0_rlc.bin");
  64 MODULE_FIRMWARE("amdgpu/gc_11_0_0_toc.bin");
  65 MODULE_FIRMWARE("amdgpu/gc_11_0_1_pfp.bin");
  66 MODULE_FIRMWARE("amdgpu/gc_11_0_1_me.bin");
  67 MODULE_FIRMWARE("amdgpu/gc_11_0_1_mec.bin");
  68 MODULE_FIRMWARE("amdgpu/gc_11_0_1_rlc.bin");
  69 MODULE_FIRMWARE("amdgpu/gc_11_0_2_pfp.bin");
  70 MODULE_FIRMWARE("amdgpu/gc_11_0_2_me.bin");
  71 MODULE_FIRMWARE("amdgpu/gc_11_0_2_mec.bin");
  72 MODULE_FIRMWARE("amdgpu/gc_11_0_2_rlc.bin");
  73
  74 static const struct soc15_reg_golden golden_settings_gc_11_0[] =
  75 {
  76         /* Pending on emulation bring up */
  77 };
  78
  79 static const struct soc15_reg_golden golden_settings_gc_11_0_0[] =
  80 {
  81         /* Pending on emulation bring up */
  82 };
  83
  84 static const struct soc15_reg_golden golden_settings_gc_rlc_spm_11_0[] =
  85 {
  86         /* Pending on emulation bring up */
  87 };
  88
  89 static const struct soc15_reg_golden golden_settings_gc_11_0_1[] =
  90 {
  91         SOC15_REG_GOLDEN_VALUE(GC, 0, regCGTT_GS_NGG_CLK_CTRL, 0x9fff8fff, 0x00000010),
  92         SOC15_REG_GOLDEN_VALUE(GC, 0, regCGTT_WD_CLK_CTRL, 0xffff8fff, 0x00000010),
  93         SOC15_REG_GOLDEN_VALUE(GC, 0, regCPF_GCR_CNTL, 0x0007ffff, 0x0000c200),
  94         SOC15_REG_GOLDEN_VALUE(GC, 0, regGL2C_CTRL3, 0xffff001b, 0x00f01988),
  95         SOC15_REG_GOLDEN_VALUE(GC, 0, regPA_CL_ENHANCE, 0xf0ffffff, 0x00880007),
  96         SOC15_REG_GOLDEN_VALUE(GC, 0, regPA_SC_ENHANCE_3, 0xfffffffd, 0x00000008),
  97         SOC15_REG_GOLDEN_VALUE(GC, 0, regPA_SC_VRS_SURFACE_CNTL_1, 0xfff891ff, 0x55480100),
  98         SOC15_REG_GOLDEN_VALUE(GC, 0, regTA_CNTL_AUX, 0xf7f7ffff, 0x01030000),
  99         SOC15_REG_GOLDEN_VALUE(GC, 0, regTCP_CNTL2, 0xfcffffff, 0x0000000a)
 100 };
 101
 102 #define DEFAULT_SH_MEM_CONFIG \
 103         ((SH_MEM_ADDRESS_MODE_64 << SH_MEM_CONFIG__ADDRESS_MODE__SHIFT) | \
 104          (SH_MEM_ALIGNMENT_MODE_UNALIGNED << SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT) | \
 105          (3 << SH_MEM_CONFIG__INITIAL_INST_PREFETCH__SHIFT))
 106
 107 static void gfx_v11_0_disable_gpa_mode(struct amdgpu_device *adev);
 108 static void gfx_v11_0_set_ring_funcs(struct amdgpu_device *adev);
 109 static void gfx_v11_0_set_irq_funcs(struct amdgpu_device *adev);
 110 static void gfx_v11_0_set_gds_init(struct amdgpu_device *adev);
 111 static void gfx_v11_0_set_rlc_funcs(struct amdgpu_device *adev);
 112 static void gfx_v11_0_set_mqd_funcs(struct amdgpu_device *adev);
 113 static void gfx_v11_0_set_imu_funcs(struct amdgpu_device *adev);
 114 static int gfx_v11_0_get_cu_info(struct amdgpu_device *adev,
 115                                  struct amdgpu_cu_info *cu_info);
 116 static uint64_t gfx_v11_0_get_gpu_clock_counter(struct amdgpu_device *adev);
 117 static void gfx_v11_0_select_se_sh(struct amdgpu_device *adev, u32 se_num,
 118                                    u32 sh_num, u32 instance);
 119 static u32 gfx_v11_0_get_wgp_active_bitmap_per_sh(struct amdgpu_device *adev);
 120
 121 static void gfx_v11_0_ring_emit_de_meta(struct amdgpu_ring *ring, bool resume);
 122 static void gfx_v11_0_ring_emit_frame_cntl(struct amdgpu_ring *ring, bool start, bool secure);
 123 static void gfx_v11_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg,
 124                                      uint32_t val);
 125 static int gfx_v11_0_wait_for_rlc_autoload_complete(struct amdgpu_device *adev);
 126 static void gfx_v11_0_ring_invalidate_tlbs(struct amdgpu_ring *ring,
 127                                            uint16_t pasid, uint32_t flush_type,
 128                                            bool all_hub, uint8_t dst_sel);
 129
 130 static void gfx11_kiq_set_resources(struct amdgpu_ring *kiq_ring, uint64_t queue_mask)
 131 {
 132         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_RESOURCES, 6));
 133         amdgpu_ring_write(kiq_ring, PACKET3_SET_RESOURCES_VMID_MASK(0) |
 134                           PACKET3_SET_RESOURCES_QUEUE_TYPE(0)); /* vmid_mask:0 queue_type:0 (KIQ) */
 135         amdgpu_ring_write(kiq_ring, lower_32_bits(queue_mask)); /* queue mask lo */
 136         amdgpu_ring_write(kiq_ring, upper_32_bits(queue_mask)); /* queue mask hi */
 137         amdgpu_ring_write(kiq_ring, 0); /* gws mask lo */
 138         amdgpu_ring_write(kiq_ring, 0); /* gws mask hi */
 139         amdgpu_ring_write(kiq_ring, 0); /* oac mask */
 140         amdgpu_ring_write(kiq_ring, 0); /* gds heap base:0, gds heap size:0 */
 141 }
 142
 143 static void gfx11_kiq_map_queues(struct amdgpu_ring *kiq_ring,
 144                                  struct amdgpu_ring *ring)
 145 {
 146         uint64_t mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj);
 147         uint64_t wptr_addr = ring->wptr_gpu_addr;
 148         uint32_t me = 0, eng_sel = 0;
 149
 150         switch (ring->funcs->type) {
 151         case AMDGPU_RING_TYPE_COMPUTE:
 152                 me = 1;
 153                 eng_sel = 0;
 154                 break;
 155         case AMDGPU_RING_TYPE_GFX:
 156                 me = 0;
 157                 eng_sel = 4;
 158                 break;
 159         case AMDGPU_RING_TYPE_MES:
 160                 me = 2;
 161                 eng_sel = 5;
 162                 break;
 163         default:
 164                 WARN_ON(1);
 165         }
 166
 167         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5));
 168         /* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/
 169         amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
 170                           PACKET3_MAP_QUEUES_QUEUE_SEL(0) | /* Queue_Sel */
 171                           PACKET3_MAP_QUEUES_VMID(0) | /* VMID */
 172                           PACKET3_MAP_QUEUES_QUEUE(ring->queue) |
 173                           PACKET3_MAP_QUEUES_PIPE(ring->pipe) |
 174                           PACKET3_MAP_QUEUES_ME((me)) |
 175                           PACKET3_MAP_QUEUES_QUEUE_TYPE(0) | /*queue_type: normal compute queue */
 176                           PACKET3_MAP_QUEUES_ALLOC_FORMAT(0) | /* alloc format: all_on_one_pipe */
 177                           PACKET3_MAP_QUEUES_ENGINE_SEL(eng_sel) |
 178                           PACKET3_MAP_QUEUES_NUM_QUEUES(1)); /* num_queues: must be 1 */
 179         amdgpu_ring_write(kiq_ring, PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index));
 180         amdgpu_ring_write(kiq_ring, lower_32_bits(mqd_addr));
 181         amdgpu_ring_write(kiq_ring, upper_32_bits(mqd_addr));
 182         amdgpu_ring_write(kiq_ring, lower_32_bits(wptr_addr));
 183         amdgpu_ring_write(kiq_ring, upper_32_bits(wptr_addr));
 184 }
 185
 186 static void gfx11_kiq_unmap_queues(struct amdgpu_ring *kiq_ring,
 187                                    struct amdgpu_ring *ring,
 188                                    enum amdgpu_unmap_queues_action action,
 189                                    u64 gpu_addr, u64 seq)
 190 {
 191         struct amdgpu_device *adev = kiq_ring->adev;
 192         uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
 193
 194         if (adev->enable_mes && !adev->gfx.kiq.ring.sched.ready) {
 195                 amdgpu_mes_unmap_legacy_queue(adev, ring, action, gpu_addr, seq);
 196                 return;
 197         }
 198
 199         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4));
 200         amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
 201                           PACKET3_UNMAP_QUEUES_ACTION(action) |
 202                           PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) |
 203                           PACKET3_UNMAP_QUEUES_ENGINE_SEL(eng_sel) |
 204                           PACKET3_UNMAP_QUEUES_NUM_QUEUES(1));
 205         amdgpu_ring_write(kiq_ring,
 206                   PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index));
 207
 208         if (action == PREEMPT_QUEUES_NO_UNMAP) {
 209                 amdgpu_ring_write(kiq_ring, lower_32_bits(gpu_addr));
 210                 amdgpu_ring_write(kiq_ring, upper_32_bits(gpu_addr));
 211                 amdgpu_ring_write(kiq_ring, seq);
 212         } else {
 213                 amdgpu_ring_write(kiq_ring, 0);
 214                 amdgpu_ring_write(kiq_ring, 0);
 215                 amdgpu_ring_write(kiq_ring, 0);
 216         }
 217 }
 218
 219 static void gfx11_kiq_query_status(struct amdgpu_ring *kiq_ring,
 220                                    struct amdgpu_ring *ring,
 221                                    u64 addr,
 222                                    u64 seq)
 223 {
 224         uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
 225
 226         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_QUERY_STATUS, 5));
 227         amdgpu_ring_write(kiq_ring,
 228                           PACKET3_QUERY_STATUS_CONTEXT_ID(0) |
 229                           PACKET3_QUERY_STATUS_INTERRUPT_SEL(0) |
 230                           PACKET3_QUERY_STATUS_COMMAND(2));
 231         amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
 232                           PACKET3_QUERY_STATUS_DOORBELL_OFFSET(ring->doorbell_index) |
 233                           PACKET3_QUERY_STATUS_ENG_SEL(eng_sel));
 234         amdgpu_ring_write(kiq_ring, lower_32_bits(addr));
 235         amdgpu_ring_write(kiq_ring, upper_32_bits(addr));
 236         amdgpu_ring_write(kiq_ring, lower_32_bits(seq));
 237         amdgpu_ring_write(kiq_ring, upper_32_bits(seq));
 238 }
 239
 240 static void gfx11_kiq_invalidate_tlbs(struct amdgpu_ring *kiq_ring,
 241                                 uint16_t pasid, uint32_t flush_type,
 242                                 bool all_hub)
 243 {
 244         gfx_v11_0_ring_invalidate_tlbs(kiq_ring, pasid, flush_type, all_hub, 1);
 245 }
 246
 247 static const struct kiq_pm4_funcs gfx_v11_0_kiq_pm4_funcs = {
 248         .kiq_set_resources = gfx11_kiq_set_resources,
 249         .kiq_map_queues = gfx11_kiq_map_queues,
 250         .kiq_unmap_queues = gfx11_kiq_unmap_queues,
 251         .kiq_query_status = gfx11_kiq_query_status,
 252         .kiq_invalidate_tlbs = gfx11_kiq_invalidate_tlbs,
 253         .set_resources_size = 8,
 254         .map_queues_size = 7,
 255         .unmap_queues_size = 6,
 256         .query_status_size = 7,
 257         .invalidate_tlbs_size = 2,
 258 };
 259
 260 static void gfx_v11_0_set_kiq_pm4_funcs(struct amdgpu_device *adev)
 261 {
 262         adev->gfx.kiq.pmf = &gfx_v11_0_kiq_pm4_funcs;
 263 }
 264
 265 static void gfx_v11_0_init_spm_golden_registers(struct amdgpu_device *adev)
 266 {
 267         switch (adev->ip_versions[GC_HWIP][0]) {
 268         case IP_VERSION(11, 0, 0):
 269                 soc15_program_register_sequence(adev,
 270                                                 golden_settings_gc_rlc_spm_11_0,
 271                                                 (const u32)ARRAY_SIZE(golden_settings_gc_rlc_spm_11_0));
 272                 break;
 273         default:
 274                 break;
 275         }
 276 }
 277
 278 static void gfx_v11_0_init_golden_registers(struct amdgpu_device *adev)
 279 {
 280         switch (adev->ip_versions[GC_HWIP][0]) {
 281         case IP_VERSION(11, 0, 0):
 282                 soc15_program_register_sequence(adev,
 283                                                 golden_settings_gc_11_0,
 284                                                 (const u32)ARRAY_SIZE(golden_settings_gc_11_0));
 285                 soc15_program_register_sequence(adev,
 286                                                 golden_settings_gc_11_0_0,
 287                                                 (const u32)ARRAY_SIZE(golden_settings_gc_11_0_0));
 288                 break;
 289         case IP_VERSION(11, 0, 1):
 290                 soc15_program_register_sequence(adev,
 291                                                 golden_settings_gc_11_0,
 292                                                 (const u32)ARRAY_SIZE(golden_settings_gc_11_0));
 293                 soc15_program_register_sequence(adev,
 294                                                 golden_settings_gc_11_0_1,
 295                                                 (const u32)ARRAY_SIZE(golden_settings_gc_11_0_1));
 296                 break;
 297         default:
 298                 break;
 299         }
 300         gfx_v11_0_init_spm_golden_registers(adev);
 301 }
 302
 303 static void gfx_v11_0_write_data_to_reg(struct amdgpu_ring *ring, int eng_sel,
 304                                        bool wc, uint32_t reg, uint32_t val)
 305 {
 306         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
 307         amdgpu_ring_write(ring, WRITE_DATA_ENGINE_SEL(eng_sel) |
 308                           WRITE_DATA_DST_SEL(0) | (wc ? WR_CONFIRM : 0));
 309         amdgpu_ring_write(ring, reg);
 310         amdgpu_ring_write(ring, 0);
 311         amdgpu_ring_write(ring, val);
 312 }
 313
 314 static void gfx_v11_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel,
 315                                   int mem_space, int opt, uint32_t addr0,
 316                                   uint32_t addr1, uint32_t ref, uint32_t mask,
 317                                   uint32_t inv)
 318 {
 319         amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
 320         amdgpu_ring_write(ring,
 321                           /* memory (1) or register (0) */
 322                           (WAIT_REG_MEM_MEM_SPACE(mem_space) |
 323                            WAIT_REG_MEM_OPERATION(opt) | /* wait */
 324                            WAIT_REG_MEM_FUNCTION(3) |  /* equal */
 325                            WAIT_REG_MEM_ENGINE(eng_sel)));
 326
 327         if (mem_space)
 328                 BUG_ON(addr0 & 0x3); /* Dword align */
 329         amdgpu_ring_write(ring, addr0);
 330         amdgpu_ring_write(ring, addr1);
 331         amdgpu_ring_write(ring, ref);
 332         amdgpu_ring_write(ring, mask);
 333         amdgpu_ring_write(ring, inv); /* poll interval */
 334 }
 335
 336 static int gfx_v11_0_ring_test_ring(struct amdgpu_ring *ring)
 337 {
 338         struct amdgpu_device *adev = ring->adev;
 339         uint32_t scratch = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG0);
 340         uint32_t tmp = 0;
 341         unsigned i;
 342         int r;
 343
 344         WREG32(scratch, 0xCAFEDEAD);
 345         r = amdgpu_ring_alloc(ring, 5);
 346         if (r) {
 347                 DRM_ERROR("amdgpu: cp failed to lock ring %d (%d).\n",
 348                           ring->idx, r);
 349                 return r;
 350         }
 351
 352         if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ) {
 353                 gfx_v11_0_ring_emit_wreg(ring, scratch, 0xDEADBEEF);
 354         } else {
 355                 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
 356                 amdgpu_ring_write(ring, scratch -
 357                                   PACKET3_SET_UCONFIG_REG_START);
 358                 amdgpu_ring_write(ring, 0xDEADBEEF);
 359         }
 360         amdgpu_ring_commit(ring);
 361
 362         for (i = 0; i < adev->usec_timeout; i++) {
 363                 tmp = RREG32(scratch);
 364                 if (tmp == 0xDEADBEEF)
 365                         break;
 366                 if (amdgpu_emu_mode == 1)
 367                         msleep(1);
 368                 else
 369                         udelay(1);
 370         }
 371
 372         if (i >= adev->usec_timeout)
 373                 r = -ETIMEDOUT;
 374         return r;
 375 }
 376
 377 static int gfx_v11_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 378 {
 379         struct amdgpu_device *adev = ring->adev;
 380         struct amdgpu_ib ib;
 381         struct dma_fence *f = NULL;
 382         unsigned index;
 383         uint64_t gpu_addr;
 384         volatile uint32_t *cpu_ptr;
 385         long r;
 386
 387         /* MES KIQ fw hasn't indirect buffer support for now */
 388         if (adev->enable_mes_kiq &&
 389             ring->funcs->type == AMDGPU_RING_TYPE_KIQ)
 390                 return 0;
 391
 392         memset(&ib, 0, sizeof(ib));
 393
 394         if (ring->is_mes_queue) {
 395                 uint32_t padding, offset;
 396
 397                 offset = amdgpu_mes_ctx_get_offs(ring, AMDGPU_MES_CTX_IB_OFFS);
 398                 padding = amdgpu_mes_ctx_get_offs(ring,
 399                                                   AMDGPU_MES_CTX_PADDING_OFFS);
 400
 401                 ib.gpu_addr = amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset);
 402                 ib.ptr = amdgpu_mes_ctx_get_offs_cpu_addr(ring, offset);
 403
 404                 gpu_addr = amdgpu_mes_ctx_get_offs_gpu_addr(ring, padding);
 405                 cpu_ptr = amdgpu_mes_ctx_get_offs_cpu_addr(ring, padding);
 406                 *cpu_ptr = cpu_to_le32(0xCAFEDEAD);
 407         } else {
 408                 r = amdgpu_device_wb_get(adev, &index);
 409                 if (r)
 410                         return r;
 411
 412                 gpu_addr = adev->wb.gpu_addr + (index * 4);
 413                 adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD);
 414                 cpu_ptr = &adev->wb.wb[index];
 415
 416                 r = amdgpu_ib_get(adev, NULL, 16, AMDGPU_IB_POOL_DIRECT, &ib);
 417                 if (r) {
 418                         DRM_ERROR("amdgpu: failed to get ib (%ld).\n", r);
 419                         goto err1;
 420                 }
 421         }
 422
 423         ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3);
 424         ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM;
 425         ib.ptr[2] = lower_32_bits(gpu_addr);
 426         ib.ptr[3] = upper_32_bits(gpu_addr);
 427         ib.ptr[4] = 0xDEADBEEF;
 428         ib.length_dw = 5;
 429
 430         r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
 431         if (r)
 432                 goto err2;
 433
 434         r = dma_fence_wait_timeout(f, false, timeout);
 435         if (r == 0) {
 436                 r = -ETIMEDOUT;
 437                 goto err2;
 438         } else if (r < 0) {
 439                 goto err2;
 440         }
 441
 442         if (le32_to_cpu(*cpu_ptr) == 0xDEADBEEF)
 443                 r = 0;
 444         else
 445                 r = -EINVAL;
 446 err2:
 447         if (!ring->is_mes_queue)
 448                 amdgpu_ib_free(adev, &ib, NULL);
 449         dma_fence_put(f);
 450 err1:
 451         if (!ring->is_mes_queue)
 452                 amdgpu_device_wb_free(adev, index);
 453         return r;
 454 }
 455
 456 static void gfx_v11_0_free_microcode(struct amdgpu_device *adev)
 457 {
 458         release_firmware(adev->gfx.pfp_fw);
 459         adev->gfx.pfp_fw = NULL;
 460         release_firmware(adev->gfx.me_fw);
 461         adev->gfx.me_fw = NULL;
 462         release_firmware(adev->gfx.rlc_fw);
 463         adev->gfx.rlc_fw = NULL;
 464         release_firmware(adev->gfx.mec_fw);
 465         adev->gfx.mec_fw = NULL;
 466
 467         kfree(adev->gfx.rlc.register_list_format);
 468 }
 469
 470 static void gfx_v11_0_init_rlc_ext_microcode(struct amdgpu_device *adev)
 471 {
 472         const struct rlc_firmware_header_v2_1 *rlc_hdr;
 473
 474         rlc_hdr = (const struct rlc_firmware_header_v2_1 *)adev->gfx.rlc_fw->data;
 475         adev->gfx.rlc_srlc_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_cntl_ucode_ver);
 476         adev->gfx.rlc_srlc_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_cntl_feature_ver);
 477         adev->gfx.rlc.save_restore_list_cntl_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_cntl_size_bytes);
 478         adev->gfx.rlc.save_restore_list_cntl = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_cntl_offset_bytes);
 479         adev->gfx.rlc_srlg_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_gpm_ucode_ver);
 480         adev->gfx.rlc_srlg_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_gpm_feature_ver);
 481         adev->gfx.rlc.save_restore_list_gpm_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_gpm_size_bytes);
 482         adev->gfx.rlc.save_restore_list_gpm = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_gpm_offset_bytes);
 483         adev->gfx.rlc_srls_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_srm_ucode_ver);
 484         adev->gfx.rlc_srls_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_srm_feature_ver);
 485         adev->gfx.rlc.save_restore_list_srm_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_srm_size_bytes);
 486         adev->gfx.rlc.save_restore_list_srm = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_srm_offset_bytes);
 487         adev->gfx.rlc.reg_list_format_direct_reg_list_length =
 488                         le32_to_cpu(rlc_hdr->reg_list_format_direct_reg_list_length);
 489 }
 490
 491 static void gfx_v11_0_init_rlc_iram_dram_microcode(struct amdgpu_device *adev)
 492 {
 493         const struct rlc_firmware_header_v2_2 *rlc_hdr;
 494
 495         rlc_hdr = (const struct rlc_firmware_header_v2_2 *)adev->gfx.rlc_fw->data;
 496         adev->gfx.rlc.rlc_iram_ucode_size_bytes = le32_to_cpu(rlc_hdr->rlc_iram_ucode_size_bytes);
 497         adev->gfx.rlc.rlc_iram_ucode = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->rlc_iram_ucode_offset_bytes);
 498         adev->gfx.rlc.rlc_dram_ucode_size_bytes = le32_to_cpu(rlc_hdr->rlc_dram_ucode_size_bytes);
 499         adev->gfx.rlc.rlc_dram_ucode = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->rlc_dram_ucode_offset_bytes);
 500 }
 501
 502 static void gfx_v11_0_init_rlcp_rlcv_microcode(struct amdgpu_device *adev)
 503 {
 504         const struct rlc_firmware_header_v2_3 *rlc_hdr;
 505
 506         rlc_hdr = (const struct rlc_firmware_header_v2_3 *)adev->gfx.rlc_fw->data;
 507         adev->gfx.rlc.rlcp_ucode_size_bytes = le32_to_cpu(rlc_hdr->rlcp_ucode_size_bytes);
 508         adev->gfx.rlc.rlcp_ucode = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->rlcp_ucode_offset_bytes);
 509         adev->gfx.rlc.rlcv_ucode_size_bytes = le32_to_cpu(rlc_hdr->rlcv_ucode_size_bytes);
 510         adev->gfx.rlc.rlcv_ucode = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->rlcv_ucode_offset_bytes);
 511 }
 512
 513 static int gfx_v11_0_init_microcode(struct amdgpu_device *adev)
 514 {
 515         char fw_name[40];
 516         char ucode_prefix[30];
 517         int err;
 518         struct amdgpu_firmware_info *info = NULL;
 519         const struct common_firmware_header *header = NULL;
 520         const struct gfx_firmware_header_v1_0 *cp_hdr;
 521         const struct gfx_firmware_header_v2_0 *cp_hdr_v2_0;
 522         const struct rlc_firmware_header_v2_0 *rlc_hdr;
 523         unsigned int *tmp = NULL;
 524         unsigned int i = 0;
 525         uint16_t version_major;
 526         uint16_t version_minor;
 527
 528         DRM_DEBUG("\n");
 529
 530         amdgpu_ucode_ip_version_decode(adev, GC_HWIP, ucode_prefix, sizeof(ucode_prefix));
 531
 532         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_pfp.bin", ucode_prefix);
 533         err = request_firmware(&adev->gfx.pfp_fw, fw_name, adev->dev);
 534         if (err)
 535                 goto out;
 536         err = amdgpu_ucode_validate(adev->gfx.pfp_fw);
 537         if (err)
 538                 goto out;
 539         /* check pfp fw hdr version to decide if enable rs64 for gfx11.*/
 540         adev->gfx.rs64_enable = amdgpu_ucode_hdr_version(
 541                                 (union amdgpu_firmware_header *)
 542                                 adev->gfx.pfp_fw->data, 2, 0);
 543         if (adev->gfx.rs64_enable) {
 544                 dev_info(adev->dev, "CP RS64 enable\n");
 545                 cp_hdr_v2_0 = (const struct gfx_firmware_header_v2_0 *)adev->gfx.pfp_fw->data;
 546                 adev->gfx.pfp_fw_version = le32_to_cpu(cp_hdr_v2_0->header.ucode_version);
 547                 adev->gfx.pfp_feature_version = le32_to_cpu(cp_hdr_v2_0->ucode_feature_version);
 548
 549         } else {
 550                 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.pfp_fw->data;
 551                 adev->gfx.pfp_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
 552                 adev->gfx.pfp_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
 553         }
 554
 555         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_me.bin", ucode_prefix);
 556         err = request_firmware(&adev->gfx.me_fw, fw_name, adev->dev);
 557         if (err)
 558                 goto out;
 559         err = amdgpu_ucode_validate(adev->gfx.me_fw);
 560         if (err)
 561                 goto out;
 562         if (adev->gfx.rs64_enable) {
 563                 cp_hdr_v2_0 = (const struct gfx_firmware_header_v2_0 *)adev->gfx.me_fw->data;
 564                 adev->gfx.me_fw_version = le32_to_cpu(cp_hdr_v2_0->header.ucode_version);
 565                 adev->gfx.me_feature_version = le32_to_cpu(cp_hdr_v2_0->ucode_feature_version);
 566
 567         } else {
 568                 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.me_fw->data;
 569                 adev->gfx.me_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
 570                 adev->gfx.me_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
 571         }
 572
 573         if (!amdgpu_sriov_vf(adev)) {
 574                 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_rlc.bin", ucode_prefix);
 575                 err = request_firmware(&adev->gfx.rlc_fw, fw_name, adev->dev);
 576                 if (err)
 577                         goto out;
 578                 err = amdgpu_ucode_validate(adev->gfx.rlc_fw);
 579                 rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
 580                 version_major = le16_to_cpu(rlc_hdr->header.header_version_major);
 581                 version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor);
 582
 583                 adev->gfx.rlc_fw_version = le32_to_cpu(rlc_hdr->header.ucode_version);
 584                 adev->gfx.rlc_feature_version = le32_to_cpu(rlc_hdr->ucode_feature_version);
 585                 adev->gfx.rlc.save_and_restore_offset =
 586                         le32_to_cpu(rlc_hdr->save_and_restore_offset);
 587                 adev->gfx.rlc.clear_state_descriptor_offset =
 588                         le32_to_cpu(rlc_hdr->clear_state_descriptor_offset);
 589                 adev->gfx.rlc.avail_scratch_ram_locations =
 590                         le32_to_cpu(rlc_hdr->avail_scratch_ram_locations);
 591                 adev->gfx.rlc.reg_restore_list_size =
 592                         le32_to_cpu(rlc_hdr->reg_restore_list_size);
 593                 adev->gfx.rlc.reg_list_format_start =
 594                         le32_to_cpu(rlc_hdr->reg_list_format_start);
 595                 adev->gfx.rlc.reg_list_format_separate_start =
 596                         le32_to_cpu(rlc_hdr->reg_list_format_separate_start);
 597                 adev->gfx.rlc.starting_offsets_start =
 598                         le32_to_cpu(rlc_hdr->starting_offsets_start);
 599                 adev->gfx.rlc.reg_list_format_size_bytes =
 600                         le32_to_cpu(rlc_hdr->reg_list_format_size_bytes);
 601                 adev->gfx.rlc.reg_list_size_bytes =
 602                         le32_to_cpu(rlc_hdr->reg_list_size_bytes);
 603                 adev->gfx.rlc.register_list_format =
 604                         kmalloc(adev->gfx.rlc.reg_list_format_size_bytes +
 605                                         adev->gfx.rlc.reg_list_size_bytes, GFP_KERNEL);
 606                 if (!adev->gfx.rlc.register_list_format) {
 607                         err = -ENOMEM;
 608                         goto out;
 609                 }
 610
 611                 tmp = (unsigned int *)((uintptr_t)rlc_hdr +
 612                                                            le32_to_cpu(rlc_hdr->reg_list_format_array_offset_bytes));
 613                 for (i = 0 ; i < (rlc_hdr->reg_list_format_size_bytes >> 2); i++)
 614                         adev->gfx.rlc.register_list_format[i] = le32_to_cpu(tmp[i]);
 615
 616                 adev->gfx.rlc.register_restore = adev->gfx.rlc.register_list_format + i;
 617
 618                 tmp = (unsigned int *)((uintptr_t)rlc_hdr +
 619                                                            le32_to_cpu(rlc_hdr->reg_list_array_offset_bytes));
 620                 for (i = 0 ; i < (rlc_hdr->reg_list_size_bytes >> 2); i++)
 621                         adev->gfx.rlc.register_restore[i] = le32_to_cpu(tmp[i]);
 622
 623                 if (version_major == 2) {
 624                         if (version_minor >= 1)
 625                                 gfx_v11_0_init_rlc_ext_microcode(adev);
 626                         if (version_minor >= 2)
 627                                 gfx_v11_0_init_rlc_iram_dram_microcode(adev);
 628                         if (version_minor == 3)
 629                                 gfx_v11_0_init_rlcp_rlcv_microcode(adev);
 630                 }
 631         }
 632
 633         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec.bin", ucode_prefix);
 634         err = request_firmware(&adev->gfx.mec_fw, fw_name, adev->dev);
 635         if (err)
 636                 goto out;
 637         err = amdgpu_ucode_validate(adev->gfx.mec_fw);
 638         if (err)
 639                 goto out;
 640         if (adev->gfx.rs64_enable) {
 641                 cp_hdr_v2_0 = (const struct gfx_firmware_header_v2_0 *)adev->gfx.mec_fw->data;
 642                 adev->gfx.mec_fw_version = le32_to_cpu(cp_hdr_v2_0->header.ucode_version);
 643                 adev->gfx.mec_feature_version = le32_to_cpu(cp_hdr_v2_0->ucode_feature_version);
 644
 645         } else {
 646                 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
 647                 adev->gfx.mec_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
 648                 adev->gfx.mec_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
 649         }
 650
 651         /* only one MEC for gfx 11.0.0. */
 652         adev->gfx.mec2_fw = NULL;
 653
 654         if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
 655                 if (adev->gfx.rs64_enable) {
 656                         cp_hdr_v2_0 = (const struct gfx_firmware_header_v2_0 *)adev->gfx.pfp_fw->data;
 657                         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_RS64_PFP];
 658                         info->ucode_id = AMDGPU_UCODE_ID_CP_RS64_PFP;
 659                         info->fw = adev->gfx.pfp_fw;
 660                         header = (const struct common_firmware_header *)info->fw->data;
 661                         adev->firmware.fw_size +=
 662                                 ALIGN(le32_to_cpu(cp_hdr_v2_0->ucode_size_bytes), PAGE_SIZE);
 663
 664                         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_RS64_PFP_P0_STACK];
 665                         info->ucode_id = AMDGPU_UCODE_ID_CP_RS64_PFP_P0_STACK;
 666                         info->fw = adev->gfx.pfp_fw;
 667                         header = (const struct common_firmware_header *)info->fw->data;
 668                         adev->firmware.fw_size +=
 669                                 ALIGN(le32_to_cpu(cp_hdr_v2_0->data_size_bytes), PAGE_SIZE);
 670
 671                         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_RS64_PFP_P1_STACK];
 672                         info->ucode_id = AMDGPU_UCODE_ID_CP_RS64_PFP_P1_STACK;
 673                         info->fw = adev->gfx.pfp_fw;
 674                         header = (const struct common_firmware_header *)info->fw->data;
 675                         adev->firmware.fw_size +=
 676                                 ALIGN(le32_to_cpu(cp_hdr_v2_0->data_size_bytes), PAGE_SIZE);
 677
 678                         cp_hdr_v2_0 = (const struct gfx_firmware_header_v2_0 *)adev->gfx.me_fw->data;
 679                         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_RS64_ME];
 680                         info->ucode_id = AMDGPU_UCODE_ID_CP_RS64_ME;
 681                         info->fw = adev->gfx.me_fw;
 682                         header = (const struct common_firmware_header *)info->fw->data;
 683                         adev->firmware.fw_size +=
 684                                 ALIGN(le32_to_cpu(cp_hdr_v2_0->ucode_size_bytes), PAGE_SIZE);
 685
 686                         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_RS64_ME_P0_STACK];
 687                         info->ucode_id = AMDGPU_UCODE_ID_CP_RS64_ME_P0_STACK;
 688                         info->fw = adev->gfx.me_fw;
 689                         header = (const struct common_firmware_header *)info->fw->data;
 690                         adev->firmware.fw_size +=
 691                                 ALIGN(le32_to_cpu(cp_hdr_v2_0->data_size_bytes), PAGE_SIZE);
 692
 693                         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_RS64_ME_P1_STACK];
 694                         info->ucode_id = AMDGPU_UCODE_ID_CP_RS64_ME_P1_STACK;
 695                         info->fw = adev->gfx.me_fw;
 696                         header = (const struct common_firmware_header *)info->fw->data;
 697                         adev->firmware.fw_size +=
 698                                 ALIGN(le32_to_cpu(cp_hdr_v2_0->data_size_bytes), PAGE_SIZE);
 699
 700                         cp_hdr_v2_0 = (const struct gfx_firmware_header_v2_0 *)adev->gfx.mec_fw->data;
 701                         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_RS64_MEC];
 702                         info->ucode_id = AMDGPU_UCODE_ID_CP_RS64_MEC;
 703                         info->fw = adev->gfx.mec_fw;
 704                         header = (const struct common_firmware_header *)info->fw->data;
 705                         adev->firmware.fw_size +=
 706                                 ALIGN(le32_to_cpu(cp_hdr_v2_0->ucode_size_bytes), PAGE_SIZE);
 707
 708                         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_RS64_MEC_P0_STACK];
 709                         info->ucode_id = AMDGPU_UCODE_ID_CP_RS64_MEC_P0_STACK;
 710                         info->fw = adev->gfx.mec_fw;
 711                         header = (const struct common_firmware_header *)info->fw->data;
 712                         adev->firmware.fw_size +=
 713                                 ALIGN(le32_to_cpu(cp_hdr_v2_0->data_size_bytes), PAGE_SIZE);
 714
 715                         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_RS64_MEC_P1_STACK];
 716                         info->ucode_id = AMDGPU_UCODE_ID_CP_RS64_MEC_P1_STACK;
 717                         info->fw = adev->gfx.mec_fw;
 718                         header = (const struct common_firmware_header *)info->fw->data;
 719                         adev->firmware.fw_size +=
 720                                 ALIGN(le32_to_cpu(cp_hdr_v2_0->data_size_bytes), PAGE_SIZE);
 721
 722                         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_RS64_MEC_P2_STACK];
 723                         info->ucode_id = AMDGPU_UCODE_ID_CP_RS64_MEC_P2_STACK;
 724                         info->fw = adev->gfx.mec_fw;
 725                         header = (const struct common_firmware_header *)info->fw->data;
 726                         adev->firmware.fw_size +=
 727                                 ALIGN(le32_to_cpu(cp_hdr_v2_0->data_size_bytes), PAGE_SIZE);
 728
 729                         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_RS64_MEC_P3_STACK];
 730                         info->ucode_id = AMDGPU_UCODE_ID_CP_RS64_MEC_P3_STACK;
 731                         info->fw = adev->gfx.mec_fw;
 732                         header = (const struct common_firmware_header *)info->fw->data;
 733                         adev->firmware.fw_size +=
 734                                 ALIGN(le32_to_cpu(cp_hdr_v2_0->data_size_bytes), PAGE_SIZE);
 735                 } else {
 736                         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_PFP];
 737                         info->ucode_id = AMDGPU_UCODE_ID_CP_PFP;
 738                         info->fw = adev->gfx.pfp_fw;
 739                         header = (const struct common_firmware_header *)info->fw->data;
 740                         adev->firmware.fw_size +=
 741                                 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
 742
 743                         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_ME];
 744                         info->ucode_id = AMDGPU_UCODE_ID_CP_ME;
 745                         info->fw = adev->gfx.me_fw;
 746                         header = (const struct common_firmware_header *)info->fw->data;
 747                         adev->firmware.fw_size +=
 748                                 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
 749
 750                         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1];
 751                         info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1;
 752                         info->fw = adev->gfx.mec_fw;
 753                         header = (const struct common_firmware_header *)info->fw->data;
 754                         cp_hdr = (const struct gfx_firmware_header_v1_0 *)info->fw->data;
 755                         adev->firmware.fw_size +=
 756                                 ALIGN(le32_to_cpu(header->ucode_size_bytes) -
 757                                       le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE);
 758
 759                         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1_JT];
 760                         info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1_JT;
 761                         info->fw = adev->gfx.mec_fw;
 762                         adev->firmware.fw_size +=
 763                                 ALIGN(le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE);
 764                 }
 765
 766                 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_G];
 767                 info->ucode_id = AMDGPU_UCODE_ID_RLC_G;
 768                 info->fw = adev->gfx.rlc_fw;
 769                 if (info->fw) {
 770                         header = (const struct common_firmware_header *)info->fw->data;
 771                         adev->firmware.fw_size +=
 772                                 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
 773                 }
 774                 if (adev->gfx.rlc.save_restore_list_gpm_size_bytes &&
 775                     adev->gfx.rlc.save_restore_list_srm_size_bytes) {
 776                         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM];
 777                         info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM;
 778                         info->fw = adev->gfx.rlc_fw;
 779                         adev->firmware.fw_size +=
 780                                 ALIGN(adev->gfx.rlc.save_restore_list_gpm_size_bytes, PAGE_SIZE);
 781
 782                         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM];
 783                         info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM;
 784                         info->fw = adev->gfx.rlc_fw;
 785                         adev->firmware.fw_size +=
 786                                 ALIGN(adev->gfx.rlc.save_restore_list_srm_size_bytes, PAGE_SIZE);
 787                 }
 788
 789                 if (adev->gfx.rlc.rlc_iram_ucode_size_bytes &&
 790                     adev->gfx.rlc.rlc_dram_ucode_size_bytes) {
 791                         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_IRAM];
 792                         info->ucode_id = AMDGPU_UCODE_ID_RLC_IRAM;
 793                         info->fw = adev->gfx.rlc_fw;
 794                         adev->firmware.fw_size +=
 795                                 ALIGN(adev->gfx.rlc.rlc_iram_ucode_size_bytes, PAGE_SIZE);
 796
 797                         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_DRAM];
 798                         info->ucode_id = AMDGPU_UCODE_ID_RLC_DRAM;
 799                         info->fw = adev->gfx.rlc_fw;
 800                         adev->firmware.fw_size +=
 801                                 ALIGN(adev->gfx.rlc.rlc_dram_ucode_size_bytes, PAGE_SIZE);
 802                 }
 803
 804                 if (adev->gfx.rlc.rlcp_ucode_size_bytes) {
 805                         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_P];
 806                         info->ucode_id = AMDGPU_UCODE_ID_RLC_P;
 807                         info->fw = adev->gfx.rlc_fw;
 808                         adev->firmware.fw_size +=
 809                                 ALIGN(adev->gfx.rlc.rlcp_ucode_size_bytes, PAGE_SIZE);
 810                 }
 811
 812                 if (adev->gfx.rlc.rlcv_ucode_size_bytes) {
 813                         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_V];
 814                         info->ucode_id = AMDGPU_UCODE_ID_RLC_V;
 815                         info->fw = adev->gfx.rlc_fw;
 816                         adev->firmware.fw_size +=
 817                                 ALIGN(adev->gfx.rlc.rlcv_ucode_size_bytes, PAGE_SIZE);
 818                 }
 819         }
 820
 821 out:
 822         if (err) {
 823                 dev_err(adev->dev,
 824                         "gfx11: Failed to load firmware \"%s\"\n",
 825                         fw_name);
 826                 release_firmware(adev->gfx.pfp_fw);
 827                 adev->gfx.pfp_fw = NULL;
 828                 release_firmware(adev->gfx.me_fw);
 829                 adev->gfx.me_fw = NULL;
 830                 release_firmware(adev->gfx.rlc_fw);
 831                 adev->gfx.rlc_fw = NULL;
 832                 release_firmware(adev->gfx.mec_fw);
 833                 adev->gfx.mec_fw = NULL;
 834         }
 835
 836         return err;
 837 }
 838
 839 static int gfx_v11_0_init_toc_microcode(struct amdgpu_device *adev)
 840 {
 841         const struct psp_firmware_header_v1_0 *toc_hdr;
 842         int err = 0;
 843         char fw_name[40];
 844         char ucode_prefix[30];
 845
 846         amdgpu_ucode_ip_version_decode(adev, GC_HWIP, ucode_prefix, sizeof(ucode_prefix));
 847
 848         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_toc.bin", ucode_prefix);
 849         err = request_firmware(&adev->psp.toc_fw, fw_name, adev->dev);
 850         if (err)
 851                 goto out;
 852
 853         err = amdgpu_ucode_validate(adev->psp.toc_fw);
 854         if (err)
 855                 goto out;
 856
 857         toc_hdr = (const struct psp_firmware_header_v1_0 *)adev->psp.toc_fw->data;
 858         adev->psp.toc.fw_version = le32_to_cpu(toc_hdr->header.ucode_version);
 859         adev->psp.toc.feature_version = le32_to_cpu(toc_hdr->sos.fw_version);
 860         adev->psp.toc.size_bytes = le32_to_cpu(toc_hdr->header.ucode_size_bytes);
 861         adev->psp.toc.start_addr = (uint8_t *)toc_hdr +
 862                                 le32_to_cpu(toc_hdr->header.ucode_array_offset_bytes);
 863         return 0;
 864 out:
 865         dev_err(adev->dev, "Failed to load TOC microcode\n");
 866         release_firmware(adev->psp.toc_fw);
 867         adev->psp.toc_fw = NULL;
 868         return err;
 869 }
 870
 871 static u32 gfx_v11_0_get_csb_size(struct amdgpu_device *adev)
 872 {
 873         u32 count = 0;
 874         const struct cs_section_def *sect = NULL;
 875         const struct cs_extent_def *ext = NULL;
 876
 877         /* begin clear state */
 878         count += 2;
 879         /* context control state */
 880         count += 3;
 881
 882         for (sect = gfx11_cs_data; sect->section != NULL; ++sect) {
 883                 for (ext = sect->section; ext->extent != NULL; ++ext) {
 884                         if (sect->id == SECT_CONTEXT)
 885                                 count += 2 + ext->reg_count;
 886                         else
 887                                 return 0;
 888                 }
 889         }
 890
 891         /* set PA_SC_TILE_STEERING_OVERRIDE */
 892         count += 3;
 893         /* end clear state */
 894         count += 2;
 895         /* clear state */
 896         count += 2;
 897
 898         return count;
 899 }
 900
 901 static void gfx_v11_0_get_csb_buffer(struct amdgpu_device *adev,
 902                                     volatile u32 *buffer)
 903 {
 904         u32 count = 0, i;
 905         const struct cs_section_def *sect = NULL;
 906         const struct cs_extent_def *ext = NULL;
 907         int ctx_reg_offset;
 908
 909         if (adev->gfx.rlc.cs_data == NULL)
 910                 return;
 911         if (buffer == NULL)
 912                 return;
 913
 914         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
 915         buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
 916
 917         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
 918         buffer[count++] = cpu_to_le32(0x80000000);
 919         buffer[count++] = cpu_to_le32(0x80000000);
 920
 921         for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) {
 922                 for (ext = sect->section; ext->extent != NULL; ++ext) {
 923                         if (sect->id == SECT_CONTEXT) {
 924                                 buffer[count++] =
 925                                         cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
 926                                 buffer[count++] = cpu_to_le32(ext->reg_index -
 927                                                 PACKET3_SET_CONTEXT_REG_START);
 928                                 for (i = 0; i < ext->reg_count; i++)
 929                                         buffer[count++] = cpu_to_le32(ext->extent[i]);
 930                         } else {
 931                                 return;
 932                         }
 933                 }
 934         }
 935
 936         ctx_reg_offset =
 937                 SOC15_REG_OFFSET(GC, 0, regPA_SC_TILE_STEERING_OVERRIDE) - PACKET3_SET_CONTEXT_REG_START;
 938         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 1));
 939         buffer[count++] = cpu_to_le32(ctx_reg_offset);
 940         buffer[count++] = cpu_to_le32(adev->gfx.config.pa_sc_tile_steering_override);
 941
 942         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
 943         buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
 944
 945         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
 946         buffer[count++] = cpu_to_le32(0);
 947 }
 948
 949 static void gfx_v11_0_rlc_fini(struct amdgpu_device *adev)
 950 {
 951         /* clear state block */
 952         amdgpu_bo_free_kernel(&adev->gfx.rlc.clear_state_obj,
 953                         &adev->gfx.rlc.clear_state_gpu_addr,
 954                         (void **)&adev->gfx.rlc.cs_ptr);
 955
 956         /* jump table block */
 957         amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj,
 958                         &adev->gfx.rlc.cp_table_gpu_addr,
 959                         (void **)&adev->gfx.rlc.cp_table_ptr);
 960 }
 961
 962 static void gfx_v11_0_init_rlcg_reg_access_ctrl(struct amdgpu_device *adev)
 963 {
 964         struct amdgpu_rlcg_reg_access_ctrl *reg_access_ctrl;
 965
 966         reg_access_ctrl = &adev->gfx.rlc.reg_access_ctrl;
 967         reg_access_ctrl->scratch_reg0 = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG0);
 968         reg_access_ctrl->scratch_reg1 = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG1);
 969         reg_access_ctrl->scratch_reg2 = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG2);
 970         reg_access_ctrl->scratch_reg3 = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG3);
 971         reg_access_ctrl->grbm_cntl = SOC15_REG_OFFSET(GC, 0, regGRBM_GFX_CNTL);
 972         reg_access_ctrl->grbm_idx = SOC15_REG_OFFSET(GC, 0, regGRBM_GFX_INDEX);
 973         reg_access_ctrl->spare_int = SOC15_REG_OFFSET(GC, 0, regRLC_SPARE_INT_0);
 974         adev->gfx.rlc.rlcg_reg_access_supported = true;
 975 }
 976
 977 static int gfx_v11_0_rlc_init(struct amdgpu_device *adev)
 978 {
 979         const struct cs_section_def *cs_data;
 980         int r;
 981
 982         adev->gfx.rlc.cs_data = gfx11_cs_data;
 983
 984         cs_data = adev->gfx.rlc.cs_data;
 985
 986         if (cs_data) {
 987                 /* init clear state block */
 988                 r = amdgpu_gfx_rlc_init_csb(adev);
 989                 if (r)
 990                         return r;
 991         }
 992
 993         /* init spm vmid with 0xf */
 994         if (adev->gfx.rlc.funcs->update_spm_vmid)
 995                 adev->gfx.rlc.funcs->update_spm_vmid(adev, 0xf);
 996
 997         return 0;
 998 }
 999
1000 static void gfx_v11_0_mec_fini(struct amdgpu_device *adev)
1001 {
1002         amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL);
1003         amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_obj, NULL, NULL);
1004         amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_data_obj, NULL, NULL);
1005 }
1006
1007 static int gfx_v11_0_me_init(struct amdgpu_device *adev)
1008 {
1009         int r;
1010
1011         bitmap_zero(adev->gfx.me.queue_bitmap, AMDGPU_MAX_GFX_QUEUES);
1012
1013         amdgpu_gfx_graphics_queue_acquire(adev);
1014
1015         r = gfx_v11_0_init_microcode(adev);
1016         if (r)
1017                 DRM_ERROR("Failed to load gfx firmware!\n");
1018
1019         return r;
1020 }
1021
1022 static int gfx_v11_0_mec_init(struct amdgpu_device *adev)
1023 {
1024         int r;
1025         u32 *hpd;
1026         size_t mec_hpd_size;
1027
1028         bitmap_zero(adev->gfx.mec.queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
1029
1030         /* take ownership of the relevant compute queues */
1031         amdgpu_gfx_compute_queue_acquire(adev);
1032         mec_hpd_size = adev->gfx.num_compute_rings * GFX11_MEC_HPD_SIZE;
1033
1034         if (mec_hpd_size) {
1035                 r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE,
1036                                               AMDGPU_GEM_DOMAIN_GTT,
1037                                               &adev->gfx.mec.hpd_eop_obj,
1038                                               &adev->gfx.mec.hpd_eop_gpu_addr,
1039                                               (void **)&hpd);
1040                 if (r) {
1041                         dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r);
1042                         gfx_v11_0_mec_fini(adev);
1043                         return r;
1044                 }
1045
1046                 memset(hpd, 0, mec_hpd_size);
1047
1048                 amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj);
1049                 amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj);
1050         }
1051
1052         return 0;
1053 }
1054
1055 static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t wave, uint32_t address)
1056 {
1057         WREG32_SOC15(GC, 0, regSQ_IND_INDEX,
1058                 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
1059                 (address << SQ_IND_INDEX__INDEX__SHIFT));
1060         return RREG32_SOC15(GC, 0, regSQ_IND_DATA);
1061 }
1062
1063 static void wave_read_regs(struct amdgpu_device *adev, uint32_t wave,
1064                            uint32_t thread, uint32_t regno,
1065                            uint32_t num, uint32_t *out)
1066 {
1067         WREG32_SOC15(GC, 0, regSQ_IND_INDEX,
1068                 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
1069                 (regno << SQ_IND_INDEX__INDEX__SHIFT) |
1070                 (thread << SQ_IND_INDEX__WORKITEM_ID__SHIFT) |
1071                 (SQ_IND_INDEX__AUTO_INCR_MASK));
1072         while (num--)
1073                 *(out++) = RREG32_SOC15(GC, 0, regSQ_IND_DATA);
1074 }
1075
1076 static void gfx_v11_0_read_wave_data(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields)
1077 {
1078         /* in gfx11 the SIMD_ID is specified as part of the INSTANCE
1079          * field when performing a select_se_sh so it should be
1080          * zero here */
1081         WARN_ON(simd != 0);
1082
1083         /* type 2 wave data */
1084         dst[(*no_fields)++] = 2;
1085         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_STATUS);
1086         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_PC_LO);
1087         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_PC_HI);
1088         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_EXEC_LO);
1089         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_EXEC_HI);
1090         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_HW_ID1);
1091         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_HW_ID2);
1092         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_GPR_ALLOC);
1093         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_LDS_ALLOC);
1094         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_TRAPSTS);
1095         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_IB_STS);
1096         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_IB_STS2);
1097         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_IB_DBG1);
1098         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_M0);
1099 }
1100
1101 static void gfx_v11_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t simd,
1102                                      uint32_t wave, uint32_t start,
1103                                      uint32_t size, uint32_t *dst)
1104 {
1105         WARN_ON(simd != 0);
1106
1107         wave_read_regs(
1108                 adev, wave, 0, start + SQIND_WAVE_SGPRS_OFFSET, size,
1109                 dst);
1110 }
1111
1112 static void gfx_v11_0_read_wave_vgprs(struct amdgpu_device *adev, uint32_t simd,
1113                                       uint32_t wave, uint32_t thread,
1114                                       uint32_t start, uint32_t size,
1115                                       uint32_t *dst)
1116 {
1117         wave_read_regs(
1118                 adev, wave, thread,
1119                 start + SQIND_WAVE_VGPRS_OFFSET, size, dst);
1120 }
1121
1122 static void gfx_v11_0_select_me_pipe_q(struct amdgpu_device *adev,
1123                                                                           u32 me, u32 pipe, u32 q, u32 vm)
1124 {
1125         soc21_grbm_select(adev, me, pipe, q, vm);
1126 }
1127
1128 static const struct amdgpu_gfx_funcs gfx_v11_0_gfx_funcs = {
1129         .get_gpu_clock_counter = &gfx_v11_0_get_gpu_clock_counter,
1130         .select_se_sh = &gfx_v11_0_select_se_sh,
1131         .read_wave_data = &gfx_v11_0_read_wave_data,
1132         .read_wave_sgprs = &gfx_v11_0_read_wave_sgprs,
1133         .read_wave_vgprs = &gfx_v11_0_read_wave_vgprs,
1134         .select_me_pipe_q = &gfx_v11_0_select_me_pipe_q,
1135         .init_spm_golden = &gfx_v11_0_init_spm_golden_registers,
1136 };
1137
1138 static int gfx_v11_0_gpu_early_init(struct amdgpu_device *adev)
1139 {
1140         adev->gfx.funcs = &gfx_v11_0_gfx_funcs;
1141
1142         switch (adev->ip_versions[GC_HWIP][0]) {
1143         case IP_VERSION(11, 0, 0):
1144         case IP_VERSION(11, 0, 2):
1145                 adev->gfx.config.max_hw_contexts = 8;
1146                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1147                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1148                 adev->gfx.config.sc_hiz_tile_fifo_size = 0;
1149                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
1150                 break;
1151         case IP_VERSION(11, 0, 1):
1152                 adev->gfx.config.max_hw_contexts = 8;
1153                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1154                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1155                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x80;
1156                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x300;
1157                 break;
1158         default:
1159                 BUG();
1160                 break;
1161         }
1162
1163         return 0;
1164 }
1165
1166 static int gfx_v11_0_gfx_ring_init(struct amdgpu_device *adev, int ring_id,
1167                                    int me, int pipe, int queue)
1168 {
1169         int r;
1170         struct amdgpu_ring *ring;
1171         unsigned int irq_type;
1172
1173         ring = &adev->gfx.gfx_ring[ring_id];
1174
1175         ring->me = me;
1176         ring->pipe = pipe;
1177         ring->queue = queue;
1178
1179         ring->ring_obj = NULL;
1180         ring->use_doorbell = true;
1181
1182         if (!ring_id)
1183                 ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1;
1184         else
1185                 ring->doorbell_index = adev->doorbell_index.gfx_ring1 << 1;
1186         sprintf(ring->name, "gfx_%d.%d.%d", ring->me, ring->pipe, ring->queue);
1187
1188         irq_type = AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP + ring->pipe;
1189         r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type,
1190                              AMDGPU_RING_PRIO_DEFAULT, NULL);
1191         if (r)
1192                 return r;
1193         return 0;
1194 }
1195
1196 static int gfx_v11_0_compute_ring_init(struct amdgpu_device *adev, int ring_id,
1197                                        int mec, int pipe, int queue)
1198 {
1199         int r;
1200         unsigned irq_type;
1201         struct amdgpu_ring *ring;
1202         unsigned int hw_prio;
1203
1204         ring = &adev->gfx.compute_ring[ring_id];
1205
1206         /* mec0 is me1 */
1207         ring->me = mec + 1;
1208         ring->pipe = pipe;
1209         ring->queue = queue;
1210
1211         ring->ring_obj = NULL;
1212         ring->use_doorbell = true;
1213         ring->doorbell_index = (adev->doorbell_index.mec_ring0 + ring_id) << 1;
1214         ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr
1215                                 + (ring_id * GFX11_MEC_HPD_SIZE);
1216         sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue);
1217
1218         irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
1219                 + ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec)
1220                 + ring->pipe;
1221         hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring) ?
1222                         AMDGPU_GFX_PIPE_PRIO_HIGH : AMDGPU_GFX_PIPE_PRIO_NORMAL;
1223         /* type-2 packets are deprecated on MEC, use type-3 instead */
1224         r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type,
1225                              hw_prio, NULL);
1226         if (r)
1227                 return r;
1228
1229         return 0;
1230 }
1231
1232 static struct {
1233         SOC21_FIRMWARE_ID       id;
1234         unsigned int            offset;
1235         unsigned int            size;
1236 } rlc_autoload_info[SOC21_FIRMWARE_ID_MAX];
1237
1238 static void gfx_v11_0_parse_rlc_toc(struct amdgpu_device *adev, void *rlc_toc)
1239 {
1240         RLC_TABLE_OF_CONTENT *ucode = rlc_toc;
1241
1242         while (ucode && (ucode->id > SOC21_FIRMWARE_ID_INVALID) &&
1243                         (ucode->id < SOC21_FIRMWARE_ID_MAX)) {
1244                 rlc_autoload_info[ucode->id].id = ucode->id;
1245                 rlc_autoload_info[ucode->id].offset = ucode->offset * 4;
1246                 rlc_autoload_info[ucode->id].size = ucode->size * 4;
1247
1248                 ucode++;
1249         }
1250 }
1251
1252 static uint32_t gfx_v11_0_calc_toc_total_size(struct amdgpu_device *adev)
1253 {
1254         uint32_t total_size = 0;
1255         SOC21_FIRMWARE_ID id;
1256
1257         gfx_v11_0_parse_rlc_toc(adev, adev->psp.toc.start_addr);
1258
1259         for (id = SOC21_FIRMWARE_ID_RLC_G_UCODE; id < SOC21_FIRMWARE_ID_MAX; id++)
1260                 total_size += rlc_autoload_info[id].size;
1261
1262         /* In case the offset in rlc toc ucode is aligned */
1263         if (total_size < rlc_autoload_info[SOC21_FIRMWARE_ID_MAX-1].offset)
1264                 total_size = rlc_autoload_info[SOC21_FIRMWARE_ID_MAX-1].offset +
1265                         rlc_autoload_info[SOC21_FIRMWARE_ID_MAX-1].size;
1266
1267         return total_size;
1268 }
1269
1270 static int gfx_v11_0_rlc_autoload_buffer_init(struct amdgpu_device *adev)
1271 {
1272         int r;
1273         uint32_t total_size;
1274
1275         total_size = gfx_v11_0_calc_toc_total_size(adev);
1276
1277         r = amdgpu_bo_create_reserved(adev, total_size, 64 * 1024,
1278                         AMDGPU_GEM_DOMAIN_VRAM,
1279                         &adev->gfx.rlc.rlc_autoload_bo,
1280                         &adev->gfx.rlc.rlc_autoload_gpu_addr,
1281                         (void **)&adev->gfx.rlc.rlc_autoload_ptr);
1282
1283         if (r) {
1284                 dev_err(adev->dev, "(%d) failed to create fw autoload bo\n", r);
1285                 return r;
1286         }
1287
1288         return 0;
1289 }
1290
1291 static void gfx_v11_0_rlc_backdoor_autoload_copy_ucode(struct amdgpu_device *adev,
1292                                               SOC21_FIRMWARE_ID id,
1293                                               const void *fw_data,
1294                                               uint32_t fw_size,
1295                                               uint32_t *fw_autoload_mask)
1296 {
1297         uint32_t toc_offset;
1298         uint32_t toc_fw_size;
1299         char *ptr = adev->gfx.rlc.rlc_autoload_ptr;
1300
1301         if (id <= SOC21_FIRMWARE_ID_INVALID || id >= SOC21_FIRMWARE_ID_MAX)
1302                 return;
1303
1304         toc_offset = rlc_autoload_info[id].offset;
1305         toc_fw_size = rlc_autoload_info[id].size;
1306
1307         if (fw_size == 0)
1308                 fw_size = toc_fw_size;
1309
1310         if (fw_size > toc_fw_size)
1311                 fw_size = toc_fw_size;
1312
1313         memcpy(ptr + toc_offset, fw_data, fw_size);
1314
1315         if (fw_size < toc_fw_size)
1316                 memset(ptr + toc_offset + fw_size, 0, toc_fw_size - fw_size);
1317
1318         if ((id != SOC21_FIRMWARE_ID_RS64_PFP) && (id != SOC21_FIRMWARE_ID_RS64_ME))
1319                 *(uint64_t *)fw_autoload_mask |= 1 << id;
1320 }
1321
1322 static void gfx_v11_0_rlc_backdoor_autoload_copy_toc_ucode(struct amdgpu_device *adev,
1323                                                         uint32_t *fw_autoload_mask)
1324 {
1325         void *data;
1326         uint32_t size;
1327         uint64_t *toc_ptr;
1328
1329         *(uint64_t *)fw_autoload_mask |= 0x1;
1330
1331         DRM_DEBUG("rlc autoload enabled fw: 0x%llx\n", *(uint64_t *)fw_autoload_mask);
1332
1333         data = adev->psp.toc.start_addr;
1334         size = rlc_autoload_info[SOC21_FIRMWARE_ID_RLC_TOC].size;
1335
1336         toc_ptr = (uint64_t *)data + size / 8 - 1;
1337         *toc_ptr = *(uint64_t *)fw_autoload_mask;
1338
1339         gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RLC_TOC,
1340                                         data, size, fw_autoload_mask);
1341 }
1342
1343 static void gfx_v11_0_rlc_backdoor_autoload_copy_gfx_ucode(struct amdgpu_device *adev,
1344                                                         uint32_t *fw_autoload_mask)
1345 {
1346         const __le32 *fw_data;
1347         uint32_t fw_size;
1348         const struct gfx_firmware_header_v1_0 *cp_hdr;
1349         const struct gfx_firmware_header_v2_0 *cpv2_hdr;
1350         const struct rlc_firmware_header_v2_0 *rlc_hdr;
1351         const struct rlc_firmware_header_v2_2 *rlcv22_hdr;
1352         uint16_t version_major, version_minor;
1353
1354         if (adev->gfx.rs64_enable) {
1355                 /* pfp ucode */
1356                 cpv2_hdr = (const struct gfx_firmware_header_v2_0 *)
1357                         adev->gfx.pfp_fw->data;
1358                 /* instruction */
1359                 fw_data = (const __le32 *)(adev->gfx.pfp_fw->data +
1360                         le32_to_cpu(cpv2_hdr->ucode_offset_bytes));
1361                 fw_size = le32_to_cpu(cpv2_hdr->ucode_size_bytes);
1362                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_PFP,
1363                                                 fw_data, fw_size, fw_autoload_mask);
1364                 /* data */
1365                 fw_data = (const __le32 *)(adev->gfx.pfp_fw->data +
1366                         le32_to_cpu(cpv2_hdr->data_offset_bytes));
1367                 fw_size = le32_to_cpu(cpv2_hdr->data_size_bytes);
1368                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_PFP_P0_STACK,
1369                                                 fw_data, fw_size, fw_autoload_mask);
1370                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_PFP_P1_STACK,
1371                                                 fw_data, fw_size, fw_autoload_mask);
1372                 /* me ucode */
1373                 cpv2_hdr = (const struct gfx_firmware_header_v2_0 *)
1374                         adev->gfx.me_fw->data;
1375                 /* instruction */
1376                 fw_data = (const __le32 *)(adev->gfx.me_fw->data +
1377                         le32_to_cpu(cpv2_hdr->ucode_offset_bytes));
1378                 fw_size = le32_to_cpu(cpv2_hdr->ucode_size_bytes);
1379                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_ME,
1380                                                 fw_data, fw_size, fw_autoload_mask);
1381                 /* data */
1382                 fw_data = (const __le32 *)(adev->gfx.me_fw->data +
1383                         le32_to_cpu(cpv2_hdr->data_offset_bytes));
1384                 fw_size = le32_to_cpu(cpv2_hdr->data_size_bytes);
1385                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_ME_P0_STACK,
1386                                                 fw_data, fw_size, fw_autoload_mask);
1387                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_ME_P1_STACK,
1388                                                 fw_data, fw_size, fw_autoload_mask);
1389                 /* mec ucode */
1390                 cpv2_hdr = (const struct gfx_firmware_header_v2_0 *)
1391                         adev->gfx.mec_fw->data;
1392                 /* instruction */
1393                 fw_data = (const __le32 *) (adev->gfx.mec_fw->data +
1394                         le32_to_cpu(cpv2_hdr->ucode_offset_bytes));
1395                 fw_size = le32_to_cpu(cpv2_hdr->ucode_size_bytes);
1396                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_MEC,
1397                                                 fw_data, fw_size, fw_autoload_mask);
1398                 /* data */
1399                 fw_data = (const __le32 *) (adev->gfx.mec_fw->data +
1400                         le32_to_cpu(cpv2_hdr->data_offset_bytes));
1401                 fw_size = le32_to_cpu(cpv2_hdr->data_size_bytes);
1402                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_MEC_P0_STACK,
1403                                                 fw_data, fw_size, fw_autoload_mask);
1404                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_MEC_P1_STACK,
1405                                                 fw_data, fw_size, fw_autoload_mask);
1406                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_MEC_P2_STACK,
1407                                                 fw_data, fw_size, fw_autoload_mask);
1408                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_MEC_P3_STACK,
1409                                                 fw_data, fw_size, fw_autoload_mask);
1410         } else {
1411                 /* pfp ucode */
1412                 cp_hdr = (const struct gfx_firmware_header_v1_0 *)
1413                         adev->gfx.pfp_fw->data;
1414                 fw_data = (const __le32 *)(adev->gfx.pfp_fw->data +
1415                                 le32_to_cpu(cp_hdr->header.ucode_array_offset_bytes));
1416                 fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes);
1417                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_CP_PFP,
1418                                                 fw_data, fw_size, fw_autoload_mask);
1419
1420                 /* me ucode */
1421                 cp_hdr = (const struct gfx_firmware_header_v1_0 *)
1422                         adev->gfx.me_fw->data;
1423                 fw_data = (const __le32 *)(adev->gfx.me_fw->data +
1424                                 le32_to_cpu(cp_hdr->header.ucode_array_offset_bytes));
1425                 fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes);
1426                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_CP_ME,
1427                                                 fw_data, fw_size, fw_autoload_mask);
1428
1429                 /* mec ucode */
1430                 cp_hdr = (const struct gfx_firmware_header_v1_0 *)
1431                         adev->gfx.mec_fw->data;
1432                 fw_data = (const __le32 *) (adev->gfx.mec_fw->data +
1433                                 le32_to_cpu(cp_hdr->header.ucode_array_offset_bytes));
1434                 fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes) -
1435                         cp_hdr->jt_size * 4;
1436                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_CP_MEC,
1437                                                 fw_data, fw_size, fw_autoload_mask);
1438         }
1439
1440         /* rlc ucode */
1441         rlc_hdr = (const struct rlc_firmware_header_v2_0 *)
1442                 adev->gfx.rlc_fw->data;
1443         fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1444                         le32_to_cpu(rlc_hdr->header.ucode_array_offset_bytes));
1445         fw_size = le32_to_cpu(rlc_hdr->header.ucode_size_bytes);
1446         gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RLC_G_UCODE,
1447                                         fw_data, fw_size, fw_autoload_mask);
1448
1449         version_major = le16_to_cpu(rlc_hdr->header.header_version_major);
1450         version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor);
1451         if (version_major == 2) {
1452                 if (version_minor >= 2) {
1453                         rlcv22_hdr = (const struct rlc_firmware_header_v2_2 *)adev->gfx.rlc_fw->data;
1454
1455                         fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1456                                         le32_to_cpu(rlcv22_hdr->rlc_iram_ucode_offset_bytes));
1457                         fw_size = le32_to_cpu(rlcv22_hdr->rlc_iram_ucode_size_bytes);
1458                         gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RLX6_UCODE,
1459                                         fw_data, fw_size, fw_autoload_mask);
1460
1461                         fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1462                                         le32_to_cpu(rlcv22_hdr->rlc_dram_ucode_offset_bytes));
1463                         fw_size = le32_to_cpu(rlcv22_hdr->rlc_dram_ucode_size_bytes);
1464                         gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RLX6_DRAM_BOOT,
1465                                         fw_data, fw_size, fw_autoload_mask);
1466                 }
1467         }
1468 }
1469
1470 static void gfx_v11_0_rlc_backdoor_autoload_copy_sdma_ucode(struct amdgpu_device *adev,
1471                                                         uint32_t *fw_autoload_mask)
1472 {
1473         const __le32 *fw_data;
1474         uint32_t fw_size;
1475         const struct sdma_firmware_header_v2_0 *sdma_hdr;
1476
1477         sdma_hdr = (const struct sdma_firmware_header_v2_0 *)
1478                 adev->sdma.instance[0].fw->data;
1479         fw_data = (const __le32 *) (adev->sdma.instance[0].fw->data +
1480                         le32_to_cpu(sdma_hdr->header.ucode_array_offset_bytes));
1481         fw_size = le32_to_cpu(sdma_hdr->ctx_ucode_size_bytes);
1482
1483         gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev,
1484                         SOC21_FIRMWARE_ID_SDMA_UCODE_TH0, fw_data, fw_size, fw_autoload_mask);
1485
1486         fw_data = (const __le32 *) (adev->sdma.instance[0].fw->data +
1487                         le32_to_cpu(sdma_hdr->ctl_ucode_offset));
1488         fw_size = le32_to_cpu(sdma_hdr->ctl_ucode_size_bytes);
1489
1490         gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev,
1491                         SOC21_FIRMWARE_ID_SDMA_UCODE_TH1, fw_data, fw_size, fw_autoload_mask);
1492 }
1493
1494 static void gfx_v11_0_rlc_backdoor_autoload_copy_mes_ucode(struct amdgpu_device *adev,
1495                                                         uint32_t *fw_autoload_mask)
1496 {
1497         const __le32 *fw_data;
1498         unsigned fw_size;
1499         const struct mes_firmware_header_v1_0 *mes_hdr;
1500         int pipe, ucode_id, data_id;
1501
1502         for (pipe = 0; pipe < 2; pipe++) {
1503                 if (pipe==0) {
1504                         ucode_id = SOC21_FIRMWARE_ID_RS64_MES_P0;
1505                         data_id  = SOC21_FIRMWARE_ID_RS64_MES_P0_STACK;
1506                 } else {
1507                         ucode_id = SOC21_FIRMWARE_ID_RS64_MES_P1;
1508                         data_id  = SOC21_FIRMWARE_ID_RS64_MES_P1_STACK;
1509                 }
1510
1511                 mes_hdr = (const struct mes_firmware_header_v1_0 *)
1512                         adev->mes.fw[pipe]->data;
1513
1514                 fw_data = (const __le32 *)(adev->mes.fw[pipe]->data +
1515                                 le32_to_cpu(mes_hdr->mes_ucode_offset_bytes));
1516                 fw_size = le32_to_cpu(mes_hdr->mes_ucode_size_bytes);
1517
1518                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev,
1519                                 ucode_id, fw_data, fw_size, fw_autoload_mask);
1520
1521                 fw_data = (const __le32 *)(adev->mes.fw[pipe]->data +
1522                                 le32_to_cpu(mes_hdr->mes_ucode_data_offset_bytes));
1523                 fw_size = le32_to_cpu(mes_hdr->mes_ucode_data_size_bytes);
1524
1525                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev,
1526                                 data_id, fw_data, fw_size, fw_autoload_mask);
1527         }
1528 }
1529
1530 static int gfx_v11_0_rlc_backdoor_autoload_enable(struct amdgpu_device *adev)
1531 {
1532         uint32_t rlc_g_offset, rlc_g_size;
1533         uint64_t gpu_addr;
1534         uint32_t autoload_fw_id[2];
1535
1536         memset(autoload_fw_id, 0, sizeof(uint32_t) * 2);
1537
1538         /* RLC autoload sequence 2: copy ucode */
1539         gfx_v11_0_rlc_backdoor_autoload_copy_sdma_ucode(adev, autoload_fw_id);
1540         gfx_v11_0_rlc_backdoor_autoload_copy_gfx_ucode(adev, autoload_fw_id);
1541         gfx_v11_0_rlc_backdoor_autoload_copy_mes_ucode(adev, autoload_fw_id);
1542         gfx_v11_0_rlc_backdoor_autoload_copy_toc_ucode(adev, autoload_fw_id);
1543
1544         rlc_g_offset = rlc_autoload_info[SOC21_FIRMWARE_ID_RLC_G_UCODE].offset;
1545         rlc_g_size = rlc_autoload_info[SOC21_FIRMWARE_ID_RLC_G_UCODE].size;
1546         gpu_addr = adev->gfx.rlc.rlc_autoload_gpu_addr + rlc_g_offset;
1547
1548         WREG32_SOC15(GC, 0, regGFX_IMU_RLC_BOOTLOADER_ADDR_HI, upper_32_bits(gpu_addr));
1549         WREG32_SOC15(GC, 0, regGFX_IMU_RLC_BOOTLOADER_ADDR_LO, lower_32_bits(gpu_addr));
1550
1551         WREG32_SOC15(GC, 0, regGFX_IMU_RLC_BOOTLOADER_SIZE, rlc_g_size);
1552
1553         /* RLC autoload sequence 3: load IMU fw */
1554         if (adev->gfx.imu.funcs->load_microcode)
1555                 adev->gfx.imu.funcs->load_microcode(adev);
1556         /* RLC autoload sequence 4 init IMU fw */
1557         if (adev->gfx.imu.funcs->setup_imu)
1558                 adev->gfx.imu.funcs->setup_imu(adev);
1559         if (adev->gfx.imu.funcs->start_imu)
1560                 adev->gfx.imu.funcs->start_imu(adev);
1561
1562         /* RLC autoload sequence 5 disable gpa mode */
1563         gfx_v11_0_disable_gpa_mode(adev);
1564
1565         return 0;
1566 }
1567
1568 static int gfx_v11_0_sw_init(void *handle)
1569 {
1570         int i, j, k, r, ring_id = 0;
1571         struct amdgpu_kiq *kiq;
1572         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1573
1574         adev->gfxhub.funcs->init(adev);
1575
1576         switch (adev->ip_versions[GC_HWIP][0]) {
1577         case IP_VERSION(11, 0, 0):
1578         case IP_VERSION(11, 0, 1):
1579         case IP_VERSION(11, 0, 2):
1580                 adev->gfx.me.num_me = 1;
1581                 adev->gfx.me.num_pipe_per_me = 1;
1582                 adev->gfx.me.num_queue_per_pipe = 1;
1583                 adev->gfx.mec.num_mec = 2;
1584                 adev->gfx.mec.num_pipe_per_mec = 4;
1585                 adev->gfx.mec.num_queue_per_pipe = 4;
1586                 break;
1587         default:
1588                 adev->gfx.me.num_me = 1;
1589                 adev->gfx.me.num_pipe_per_me = 1;
1590                 adev->gfx.me.num_queue_per_pipe = 1;
1591                 adev->gfx.mec.num_mec = 1;
1592                 adev->gfx.mec.num_pipe_per_mec = 4;
1593                 adev->gfx.mec.num_queue_per_pipe = 8;
1594                 break;
1595         }
1596
1597         /* EOP Event */
1598         r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GRBM_CP,
1599                               GFX_11_0_0__SRCID__CP_EOP_INTERRUPT,
1600                               &adev->gfx.eop_irq);
1601         if (r)
1602                 return r;
1603
1604         /* Privileged reg */
1605         r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GRBM_CP,
1606                               GFX_11_0_0__SRCID__CP_PRIV_REG_FAULT,
1607                               &adev->gfx.priv_reg_irq);
1608         if (r)
1609                 return r;
1610
1611         /* Privileged inst */
1612         r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GRBM_CP,
1613                               GFX_11_0_0__SRCID__CP_PRIV_INSTR_FAULT,
1614                               &adev->gfx.priv_inst_irq);
1615         if (r)
1616                 return r;
1617
1618         adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE;
1619
1620         if (adev->gfx.imu.funcs) {
1621                 if (adev->gfx.imu.funcs->init_microcode) {
1622                         r = adev->gfx.imu.funcs->init_microcode(adev);
1623                         if (r)
1624                                 DRM_ERROR("Failed to load imu firmware!\n");
1625                 }
1626         }
1627
1628         r = gfx_v11_0_me_init(adev);
1629         if (r)
1630                 return r;
1631
1632         r = gfx_v11_0_rlc_init(adev);
1633         if (r) {
1634                 DRM_ERROR("Failed to init rlc BOs!\n");
1635                 return r;
1636         }
1637
1638         r = gfx_v11_0_mec_init(adev);
1639         if (r) {
1640                 DRM_ERROR("Failed to init MEC BOs!\n");
1641                 return r;
1642         }
1643
1644         /* set up the gfx ring */
1645         for (i = 0; i < adev->gfx.me.num_me; i++) {
1646                 for (j = 0; j < adev->gfx.me.num_queue_per_pipe; j++) {
1647                         for (k = 0; k < adev->gfx.me.num_pipe_per_me; k++) {
1648                                 if (!amdgpu_gfx_is_me_queue_enabled(adev, i, k, j))
1649                                         continue;
1650
1651                                 r = gfx_v11_0_gfx_ring_init(adev, ring_id,
1652                                                             i, k, j);
1653                                 if (r)
1654                                         return r;
1655                                 ring_id++;
1656                         }
1657                 }
1658         }
1659
1660         ring_id = 0;
1661         /* set up the compute queues - allocate horizontally across pipes */
1662         for (i = 0; i < adev->gfx.mec.num_mec; ++i) {
1663                 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {
1664                         for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {
1665                                 if (!amdgpu_gfx_is_mec_queue_enabled(adev, i, k,
1666                                                                      j))
1667                                         continue;
1668
1669                                 r = gfx_v11_0_compute_ring_init(adev, ring_id,
1670                                                                 i, k, j);
1671                                 if (r)
1672                                         return r;
1673
1674                                 ring_id++;
1675                         }
1676                 }
1677         }
1678
1679         if (!adev->enable_mes_kiq) {
1680                 r = amdgpu_gfx_kiq_init(adev, GFX11_MEC_HPD_SIZE);
1681                 if (r) {
1682                         DRM_ERROR("Failed to init KIQ BOs!\n");
1683                         return r;
1684                 }
1685
1686                 kiq = &adev->gfx.kiq;
1687                 r = amdgpu_gfx_kiq_init_ring(adev, &kiq->ring, &kiq->irq);
1688                 if (r)
1689                         return r;
1690         }
1691
1692         r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct v11_compute_mqd));
1693         if (r)
1694                 return r;
1695
1696         /* allocate visible FB for rlc auto-loading fw */
1697         if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) {
1698                 r = gfx_v11_0_init_toc_microcode(adev);
1699                 if (r)
1700                         dev_err(adev->dev, "Failed to load toc firmware!\n");
1701                 r = gfx_v11_0_rlc_autoload_buffer_init(adev);
1702                 if (r)
1703                         return r;
1704         }
1705
1706         r = gfx_v11_0_gpu_early_init(adev);
1707         if (r)
1708                 return r;
1709
1710         return 0;
1711 }
1712
1713 static void gfx_v11_0_pfp_fini(struct amdgpu_device *adev)
1714 {
1715         amdgpu_bo_free_kernel(&adev->gfx.pfp.pfp_fw_obj,
1716                               &adev->gfx.pfp.pfp_fw_gpu_addr,
1717                               (void **)&adev->gfx.pfp.pfp_fw_ptr);
1718
1719         amdgpu_bo_free_kernel(&adev->gfx.pfp.pfp_fw_data_obj,
1720                               &adev->gfx.pfp.pfp_fw_data_gpu_addr,
1721                               (void **)&adev->gfx.pfp.pfp_fw_data_ptr);
1722 }
1723
1724 static void gfx_v11_0_me_fini(struct amdgpu_device *adev)
1725 {
1726         amdgpu_bo_free_kernel(&adev->gfx.me.me_fw_obj,
1727                               &adev->gfx.me.me_fw_gpu_addr,
1728                               (void **)&adev->gfx.me.me_fw_ptr);
1729
1730         amdgpu_bo_free_kernel(&adev->gfx.me.me_fw_data_obj,
1731                                &adev->gfx.me.me_fw_data_gpu_addr,
1732                                (void **)&adev->gfx.me.me_fw_data_ptr);
1733 }
1734
1735 static void gfx_v11_0_rlc_autoload_buffer_fini(struct amdgpu_device *adev)
1736 {
1737         amdgpu_bo_free_kernel(&adev->gfx.rlc.rlc_autoload_bo,
1738                         &adev->gfx.rlc.rlc_autoload_gpu_addr,
1739                         (void **)&adev->gfx.rlc.rlc_autoload_ptr);
1740 }
1741
1742 static int gfx_v11_0_sw_fini(void *handle)
1743 {
1744         int i;
1745         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1746
1747         for (i = 0; i < adev->gfx.num_gfx_rings; i++)
1748                 amdgpu_ring_fini(&adev->gfx.gfx_ring[i]);
1749         for (i = 0; i < adev->gfx.num_compute_rings; i++)
1750                 amdgpu_ring_fini(&adev->gfx.compute_ring[i]);
1751
1752         amdgpu_gfx_mqd_sw_fini(adev);
1753
1754         if (!adev->enable_mes_kiq) {
1755                 amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq.ring);
1756                 amdgpu_gfx_kiq_fini(adev);
1757         }
1758
1759         gfx_v11_0_pfp_fini(adev);
1760         gfx_v11_0_me_fini(adev);
1761         gfx_v11_0_rlc_fini(adev);
1762         gfx_v11_0_mec_fini(adev);
1763
1764         if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO)
1765                 gfx_v11_0_rlc_autoload_buffer_fini(adev);
1766
1767         gfx_v11_0_free_microcode(adev);
1768
1769         return 0;
1770 }
1771
1772 static void gfx_v11_0_select_se_sh(struct amdgpu_device *adev, u32 se_num,
1773                                    u32 sh_num, u32 instance)
1774 {
1775         u32 data;
1776
1777         if (instance == 0xffffffff)
1778                 data = REG_SET_FIELD(0, GRBM_GFX_INDEX,
1779                                      INSTANCE_BROADCAST_WRITES, 1);
1780         else
1781                 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX,
1782                                      instance);
1783
1784         if (se_num == 0xffffffff)
1785                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES,
1786                                      1);
1787         else
1788                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num);
1789
1790         if (sh_num == 0xffffffff)
1791                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SA_BROADCAST_WRITES,
1792                                      1);
1793         else
1794                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SA_INDEX, sh_num);
1795
1796         WREG32_SOC15(GC, 0, regGRBM_GFX_INDEX, data);
1797 }
1798
1799 static u32 gfx_v11_0_get_rb_active_bitmap(struct amdgpu_device *adev)
1800 {
1801         u32 data, mask;
1802
1803         data = RREG32_SOC15(GC, 0, regCC_RB_BACKEND_DISABLE);
1804         data |= RREG32_SOC15(GC, 0, regGC_USER_RB_BACKEND_DISABLE);
1805
1806         data &= CC_RB_BACKEND_DISABLE__BACKEND_DISABLE_MASK;
1807         data >>= GC_USER_RB_BACKEND_DISABLE__BACKEND_DISABLE__SHIFT;
1808
1809         mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se /
1810                                          adev->gfx.config.max_sh_per_se);
1811
1812         return (~data) & mask;
1813 }
1814
1815 static void gfx_v11_0_setup_rb(struct amdgpu_device *adev)
1816 {
1817         int i, j;
1818         u32 data;
1819         u32 active_rbs = 0;
1820         u32 rb_bitmap_width_per_sh = adev->gfx.config.max_backends_per_se /
1821                                         adev->gfx.config.max_sh_per_se;
1822
1823         mutex_lock(&adev->grbm_idx_mutex);
1824         for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
1825                 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
1826                         gfx_v11_0_select_se_sh(adev, i, j, 0xffffffff);
1827                         data = gfx_v11_0_get_rb_active_bitmap(adev);
1828                         active_rbs |= data << ((i * adev->gfx.config.max_sh_per_se + j) *
1829                                                rb_bitmap_width_per_sh);
1830                 }
1831         }
1832         gfx_v11_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
1833         mutex_unlock(&adev->grbm_idx_mutex);
1834
1835         adev->gfx.config.backend_enable_mask = active_rbs;
1836         adev->gfx.config.num_rbs = hweight32(active_rbs);
1837 }
1838
1839 #define DEFAULT_SH_MEM_BASES    (0x6000)
1840 #define LDS_APP_BASE           0x1
1841 #define SCRATCH_APP_BASE       0x2
1842
1843 static void gfx_v11_0_init_compute_vmid(struct amdgpu_device *adev)
1844 {
1845         int i;
1846         uint32_t sh_mem_bases;
1847         uint32_t data;
1848
1849         /*
1850          * Configure apertures:
1851          * LDS:         0x60000000'00000000 - 0x60000001'00000000 (4GB)
1852          * Scratch:     0x60000001'00000000 - 0x60000002'00000000 (4GB)
1853          * GPUVM:       0x60010000'00000000 - 0x60020000'00000000 (1TB)
1854          */
1855         sh_mem_bases = (LDS_APP_BASE << SH_MEM_BASES__SHARED_BASE__SHIFT) |
1856                         SCRATCH_APP_BASE;
1857
1858         mutex_lock(&adev->srbm_mutex);
1859         for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
1860                 soc21_grbm_select(adev, 0, 0, 0, i);
1861                 /* CP and shaders */
1862                 WREG32_SOC15(GC, 0, regSH_MEM_CONFIG, DEFAULT_SH_MEM_CONFIG);
1863                 WREG32_SOC15(GC, 0, regSH_MEM_BASES, sh_mem_bases);
1864
1865                 /* Enable trap for each kfd vmid. */
1866                 data = RREG32(SOC15_REG_OFFSET(GC, 0, regSPI_GDBG_PER_VMID_CNTL));
1867                 data = REG_SET_FIELD(data, SPI_GDBG_PER_VMID_CNTL, TRAP_EN, 1);
1868         }
1869         soc21_grbm_select(adev, 0, 0, 0, 0);
1870         mutex_unlock(&adev->srbm_mutex);
1871
1872         /* Initialize all compute VMIDs to have no GDS, GWS, or OA
1873            acccess. These should be enabled by FW for target VMIDs. */
1874         for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
1875                 WREG32_SOC15_OFFSET(GC, 0, regGDS_VMID0_BASE, 2 * i, 0);
1876                 WREG32_SOC15_OFFSET(GC, 0, regGDS_VMID0_SIZE, 2 * i, 0);
1877                 WREG32_SOC15_OFFSET(GC, 0, regGDS_GWS_VMID0, i, 0);
1878                 WREG32_SOC15_OFFSET(GC, 0, regGDS_OA_VMID0, i, 0);
1879         }
1880 }
1881
1882 static void gfx_v11_0_init_gds_vmid(struct amdgpu_device *adev)
1883 {
1884         int vmid;
1885
1886         /*
1887          * Initialize all compute and user-gfx VMIDs to have no GDS, GWS, or OA
1888          * access. Compute VMIDs should be enabled by FW for target VMIDs,
1889          * the driver can enable them for graphics. VMID0 should maintain
1890          * access so that HWS firmware can save/restore entries.
1891          */
1892         for (vmid = 1; vmid < 16; vmid++) {
1893                 WREG32_SOC15_OFFSET(GC, 0, regGDS_VMID0_BASE, 2 * vmid, 0);
1894                 WREG32_SOC15_OFFSET(GC, 0, regGDS_VMID0_SIZE, 2 * vmid, 0);
1895                 WREG32_SOC15_OFFSET(GC, 0, regGDS_GWS_VMID0, vmid, 0);
1896                 WREG32_SOC15_OFFSET(GC, 0, regGDS_OA_VMID0, vmid, 0);
1897         }
1898 }
1899
1900 static void gfx_v11_0_tcp_harvest(struct amdgpu_device *adev)
1901 {
1902         /* TODO: harvest feature to be added later. */
1903 }
1904
1905 static void gfx_v11_0_get_tcc_info(struct amdgpu_device *adev)
1906 {
1907         /* TCCs are global (not instanced). */
1908         uint32_t tcc_disable = RREG32_SOC15(GC, 0, regCGTS_TCC_DISABLE) |
1909                                RREG32_SOC15(GC, 0, regCGTS_USER_TCC_DISABLE);
1910
1911         adev->gfx.config.tcc_disabled_mask =
1912                 REG_GET_FIELD(tcc_disable, CGTS_TCC_DISABLE, TCC_DISABLE) |
1913                 (REG_GET_FIELD(tcc_disable, CGTS_TCC_DISABLE, HI_TCC_DISABLE) << 16);
1914 }
1915
1916 static void gfx_v11_0_constants_init(struct amdgpu_device *adev)
1917 {
1918         u32 tmp;
1919         int i;
1920
1921         WREG32_FIELD15_PREREG(GC, 0, GRBM_CNTL, READ_TIMEOUT, 0xff);
1922
1923         gfx_v11_0_setup_rb(adev);
1924         gfx_v11_0_get_cu_info(adev, &adev->gfx.cu_info);
1925         gfx_v11_0_get_tcc_info(adev);
1926         adev->gfx.config.pa_sc_tile_steering_override = 0;
1927
1928         /* XXX SH_MEM regs */
1929         /* where to put LDS, scratch, GPUVM in FSA64 space */
1930         mutex_lock(&adev->srbm_mutex);
1931         for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB_0].num_ids; i++) {
1932                 soc21_grbm_select(adev, 0, 0, 0, i);
1933                 /* CP and shaders */
1934                 WREG32_SOC15(GC, 0, regSH_MEM_CONFIG, DEFAULT_SH_MEM_CONFIG);
1935                 if (i != 0) {
1936                         tmp = REG_SET_FIELD(0, SH_MEM_BASES, PRIVATE_BASE,
1937                                 (adev->gmc.private_aperture_start >> 48));
1938                         tmp = REG_SET_FIELD(tmp, SH_MEM_BASES, SHARED_BASE,
1939                                 (adev->gmc.shared_aperture_start >> 48));
1940                         WREG32_SOC15(GC, 0, regSH_MEM_BASES, tmp);
1941                 }
1942         }
1943         soc21_grbm_select(adev, 0, 0, 0, 0);
1944
1945         mutex_unlock(&adev->srbm_mutex);
1946
1947         gfx_v11_0_init_compute_vmid(adev);
1948         gfx_v11_0_init_gds_vmid(adev);
1949 }
1950
1951 static void gfx_v11_0_enable_gui_idle_interrupt(struct amdgpu_device *adev,
1952                                                bool enable)
1953 {
1954         u32 tmp;
1955
1956         if (amdgpu_sriov_vf(adev))
1957                 return;
1958
1959         tmp = RREG32_SOC15(GC, 0, regCP_INT_CNTL_RING0);
1960
1961         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE,
1962                             enable ? 1 : 0);
1963         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE,
1964                             enable ? 1 : 0);
1965         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE,
1966                             enable ? 1 : 0);
1967         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE,
1968                             enable ? 1 : 0);
1969
1970         WREG32_SOC15(GC, 0, regCP_INT_CNTL_RING0, tmp);
1971 }
1972
1973 static int gfx_v11_0_init_csb(struct amdgpu_device *adev)
1974 {
1975         adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr);
1976
1977         WREG32_SOC15(GC, 0, regRLC_CSIB_ADDR_HI,
1978                         adev->gfx.rlc.clear_state_gpu_addr >> 32);
1979         WREG32_SOC15(GC, 0, regRLC_CSIB_ADDR_LO,
1980                         adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc);
1981         WREG32_SOC15(GC, 0, regRLC_CSIB_LENGTH, adev->gfx.rlc.clear_state_size);
1982
1983         return 0;
1984 }
1985
1986 void gfx_v11_0_rlc_stop(struct amdgpu_device *adev)
1987 {
1988         u32 tmp = RREG32_SOC15(GC, 0, regRLC_CNTL);
1989
1990         tmp = REG_SET_FIELD(tmp, RLC_CNTL, RLC_ENABLE_F32, 0);
1991         WREG32_SOC15(GC, 0, regRLC_CNTL, tmp);
1992 }
1993
1994 static void gfx_v11_0_rlc_reset(struct amdgpu_device *adev)
1995 {
1996         WREG32_FIELD15_PREREG(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
1997         udelay(50);
1998         WREG32_FIELD15_PREREG(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 0);
1999         udelay(50);
2000 }
2001
2002 static void gfx_v11_0_rlc_smu_handshake_cntl(struct amdgpu_device *adev,
2003                                              bool enable)
2004 {
2005         uint32_t rlc_pg_cntl;
2006
2007         rlc_pg_cntl = RREG32_SOC15(GC, 0, regRLC_PG_CNTL);
2008
2009         if (!enable) {
2010                 /* RLC_PG_CNTL[23] = 0 (default)
2011                  * RLC will wait for handshake acks with SMU
2012                  * GFXOFF will be enabled
2013                  * RLC_PG_CNTL[23] = 1
2014                  * RLC will not issue any message to SMU
2015                  * hence no handshake between SMU & RLC
2016                  * GFXOFF will be disabled
2017                  */
2018                 rlc_pg_cntl |= RLC_PG_CNTL__SMU_HANDSHAKE_DISABLE_MASK;
2019         } else
2020                 rlc_pg_cntl &= ~RLC_PG_CNTL__SMU_HANDSHAKE_DISABLE_MASK;
2021         WREG32_SOC15(GC, 0, regRLC_PG_CNTL, rlc_pg_cntl);
2022 }
2023
2024 static void gfx_v11_0_rlc_start(struct amdgpu_device *adev)
2025 {
2026         /* TODO: enable rlc & smu handshake until smu
2027          * and gfxoff feature works as expected */
2028         if (!(amdgpu_pp_feature_mask & PP_GFXOFF_MASK))
2029                 gfx_v11_0_rlc_smu_handshake_cntl(adev, false);
2030
2031         WREG32_FIELD15_PREREG(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 1);
2032         udelay(50);
2033 }
2034
2035 static void gfx_v11_0_rlc_enable_srm(struct amdgpu_device *adev)
2036 {
2037         uint32_t tmp;
2038
2039         /* enable Save Restore Machine */
2040         tmp = RREG32(SOC15_REG_OFFSET(GC, 0, regRLC_SRM_CNTL));
2041         tmp |= RLC_SRM_CNTL__AUTO_INCR_ADDR_MASK;
2042         tmp |= RLC_SRM_CNTL__SRM_ENABLE_MASK;
2043         WREG32(SOC15_REG_OFFSET(GC, 0, regRLC_SRM_CNTL), tmp);
2044 }
2045
2046 static void gfx_v11_0_load_rlcg_microcode(struct amdgpu_device *adev)
2047 {
2048         const struct rlc_firmware_header_v2_0 *hdr;
2049         const __le32 *fw_data;
2050         unsigned i, fw_size;
2051
2052         hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
2053         fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
2054                            le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2055         fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
2056
2057         WREG32_SOC15(GC, 0, regRLC_GPM_UCODE_ADDR,
2058                      RLCG_UCODE_LOADING_START_ADDRESS);
2059
2060         for (i = 0; i < fw_size; i++)
2061                 WREG32_SOC15(GC, 0, regRLC_GPM_UCODE_DATA,
2062                              le32_to_cpup(fw_data++));
2063
2064         WREG32_SOC15(GC, 0, regRLC_GPM_UCODE_ADDR, adev->gfx.rlc_fw_version);
2065 }
2066
2067 static void gfx_v11_0_load_rlc_iram_dram_microcode(struct amdgpu_device *adev)
2068 {
2069         const struct rlc_firmware_header_v2_2 *hdr;
2070         const __le32 *fw_data;
2071         unsigned i, fw_size;
2072         u32 tmp;
2073
2074         hdr = (const struct rlc_firmware_header_v2_2 *)adev->gfx.rlc_fw->data;
2075
2076         fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
2077                         le32_to_cpu(hdr->rlc_iram_ucode_offset_bytes));
2078         fw_size = le32_to_cpu(hdr->rlc_iram_ucode_size_bytes) / 4;
2079
2080         WREG32_SOC15(GC, 0, regRLC_LX6_IRAM_ADDR, 0);
2081
2082         for (i = 0; i < fw_size; i++) {
2083                 if ((amdgpu_emu_mode == 1) && (i % 100 == 99))
2084                         msleep(1);
2085                 WREG32_SOC15(GC, 0, regRLC_LX6_IRAM_DATA,
2086                                 le32_to_cpup(fw_data++));
2087         }
2088
2089         WREG32_SOC15(GC, 0, regRLC_LX6_IRAM_ADDR, adev->gfx.rlc_fw_version);
2090
2091         fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
2092                         le32_to_cpu(hdr->rlc_dram_ucode_offset_bytes));
2093         fw_size = le32_to_cpu(hdr->rlc_dram_ucode_size_bytes) / 4;
2094
2095         WREG32_SOC15(GC, 0, regRLC_LX6_DRAM_ADDR, 0);
2096         for (i = 0; i < fw_size; i++) {
2097                 if ((amdgpu_emu_mode == 1) && (i % 100 == 99))
2098                         msleep(1);
2099                 WREG32_SOC15(GC, 0, regRLC_LX6_DRAM_DATA,
2100                                 le32_to_cpup(fw_data++));
2101         }
2102
2103         WREG32_SOC15(GC, 0, regRLC_LX6_IRAM_ADDR, adev->gfx.rlc_fw_version);
2104
2105         tmp = RREG32_SOC15(GC, 0, regRLC_LX6_CNTL);
2106         tmp = REG_SET_FIELD(tmp, RLC_LX6_CNTL, PDEBUG_ENABLE, 1);
2107         tmp = REG_SET_FIELD(tmp, RLC_LX6_CNTL, BRESET, 0);
2108         WREG32_SOC15(GC, 0, regRLC_LX6_CNTL, tmp);
2109 }
2110
2111 static void gfx_v11_0_load_rlcp_rlcv_microcode(struct amdgpu_device *adev)
2112 {
2113         const struct rlc_firmware_header_v2_3 *hdr;
2114         const __le32 *fw_data;
2115         unsigned i, fw_size;
2116         u32 tmp;
2117
2118         hdr = (const struct rlc_firmware_header_v2_3 *)adev->gfx.rlc_fw->data;
2119
2120         fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
2121                         le32_to_cpu(hdr->rlcp_ucode_offset_bytes));
2122         fw_size = le32_to_cpu(hdr->rlcp_ucode_size_bytes) / 4;
2123
2124         WREG32_SOC15(GC, 0, regRLC_PACE_UCODE_ADDR, 0);
2125
2126         for (i = 0; i < fw_size; i++) {
2127                 if ((amdgpu_emu_mode == 1) && (i % 100 == 99))
2128                         msleep(1);
2129                 WREG32_SOC15(GC, 0, regRLC_PACE_UCODE_DATA,
2130                                 le32_to_cpup(fw_data++));
2131         }
2132
2133         WREG32_SOC15(GC, 0, regRLC_PACE_UCODE_ADDR, adev->gfx.rlc_fw_version);
2134
2135         tmp = RREG32_SOC15(GC, 0, regRLC_GPM_THREAD_ENABLE);
2136         tmp = REG_SET_FIELD(tmp, RLC_GPM_THREAD_ENABLE, THREAD1_ENABLE, 1);
2137         WREG32_SOC15(GC, 0, regRLC_GPM_THREAD_ENABLE, tmp);
2138
2139         fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
2140                         le32_to_cpu(hdr->rlcv_ucode_offset_bytes));
2141         fw_size = le32_to_cpu(hdr->rlcv_ucode_size_bytes) / 4;
2142
2143         WREG32_SOC15(GC, 0, regRLC_GPU_IOV_UCODE_ADDR, 0);
2144
2145         for (i = 0; i < fw_size; i++) {
2146                 if ((amdgpu_emu_mode == 1) && (i % 100 == 99))
2147                         msleep(1);
2148                 WREG32_SOC15(GC, 0, regRLC_GPU_IOV_UCODE_DATA,
2149                                 le32_to_cpup(fw_data++));
2150         }
2151
2152         WREG32_SOC15(GC, 0, regRLC_GPU_IOV_UCODE_ADDR, adev->gfx.rlc_fw_version);
2153
2154         tmp = RREG32_SOC15(GC, 0, regRLC_GPU_IOV_F32_CNTL);
2155         tmp = REG_SET_FIELD(tmp, RLC_GPU_IOV_F32_CNTL, ENABLE, 1);
2156         WREG32_SOC15(GC, 0, regRLC_GPU_IOV_F32_CNTL, tmp);
2157 }
2158
2159 static int gfx_v11_0_rlc_load_microcode(struct amdgpu_device *adev)
2160 {
2161         const struct rlc_firmware_header_v2_0 *hdr;
2162         uint16_t version_major;
2163         uint16_t version_minor;
2164
2165         if (!adev->gfx.rlc_fw)
2166                 return -EINVAL;
2167
2168         hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
2169         amdgpu_ucode_print_rlc_hdr(&hdr->header);
2170
2171         version_major = le16_to_cpu(hdr->header.header_version_major);
2172         version_minor = le16_to_cpu(hdr->header.header_version_minor);
2173
2174         if (version_major == 2) {
2175                 gfx_v11_0_load_rlcg_microcode(adev);
2176                 if (amdgpu_dpm == 1) {
2177                         if (version_minor >= 2)
2178                                 gfx_v11_0_load_rlc_iram_dram_microcode(adev);
2179                         if (version_minor == 3)
2180                                 gfx_v11_0_load_rlcp_rlcv_microcode(adev);
2181                 }
2182
2183                 return 0;
2184         }
2185
2186         return -EINVAL;
2187 }
2188
2189 static int gfx_v11_0_rlc_resume(struct amdgpu_device *adev)
2190 {
2191         int r;
2192
2193         if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
2194                 gfx_v11_0_init_csb(adev);
2195
2196                 if (!amdgpu_sriov_vf(adev)) /* enable RLC SRM */
2197                         gfx_v11_0_rlc_enable_srm(adev);
2198         } else {
2199                 if (amdgpu_sriov_vf(adev)) {
2200                         gfx_v11_0_init_csb(adev);
2201                         return 0;
2202                 }
2203
2204                 adev->gfx.rlc.funcs->stop(adev);
2205
2206                 /* disable CG */
2207                 WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL, 0);
2208
2209                 /* disable PG */
2210                 WREG32_SOC15(GC, 0, regRLC_PG_CNTL, 0);
2211
2212                 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
2213                         /* legacy rlc firmware loading */
2214                         r = gfx_v11_0_rlc_load_microcode(adev);
2215                         if (r)
2216                                 return r;
2217                 }
2218
2219                 gfx_v11_0_init_csb(adev);
2220
2221                 adev->gfx.rlc.funcs->start(adev);
2222         }
2223         return 0;
2224 }
2225
2226 static int gfx_v11_0_config_me_cache(struct amdgpu_device *adev, uint64_t addr)
2227 {
2228         uint32_t usec_timeout = 50000;  /* wait for 50ms */
2229         uint32_t tmp;
2230         int i;
2231
2232         /* Trigger an invalidation of the L1 instruction caches */
2233         tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
2234         tmp = REG_SET_FIELD(tmp, CP_ME_IC_OP_CNTL, INVALIDATE_CACHE, 1);
2235         WREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL, tmp);
2236
2237         /* Wait for invalidation complete */
2238         for (i = 0; i < usec_timeout; i++) {
2239                 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
2240                 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL,
2241                                         INVALIDATE_CACHE_COMPLETE))
2242                         break;
2243                 udelay(1);
2244         }
2245
2246         if (i >= usec_timeout) {
2247                 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2248                 return -EINVAL;
2249         }
2250
2251         if (amdgpu_emu_mode == 1)
2252                 adev->hdp.funcs->flush_hdp(adev, NULL);
2253
2254         tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL);
2255         tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, VMID, 0);
2256         tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, CACHE_POLICY, 0);
2257         tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, EXE_DISABLE, 0);
2258         tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, ADDRESS_CLAMP, 1);
2259         WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL, tmp);
2260
2261         /* Program me ucode address into intruction cache address register */
2262         WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_LO,
2263                         lower_32_bits(addr) & 0xFFFFF000);
2264         WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_HI,
2265                         upper_32_bits(addr));
2266
2267         return 0;
2268 }
2269
2270 static int gfx_v11_0_config_pfp_cache(struct amdgpu_device *adev, uint64_t addr)
2271 {
2272         uint32_t usec_timeout = 50000;  /* wait for 50ms */
2273         uint32_t tmp;
2274         int i;
2275
2276         /* Trigger an invalidation of the L1 instruction caches */
2277         tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
2278         tmp = REG_SET_FIELD(tmp, CP_PFP_IC_OP_CNTL, INVALIDATE_CACHE, 1);
2279         WREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL, tmp);
2280
2281         /* Wait for invalidation complete */
2282         for (i = 0; i < usec_timeout; i++) {
2283                 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
2284                 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL,
2285                                         INVALIDATE_CACHE_COMPLETE))
2286                         break;
2287                 udelay(1);
2288         }
2289
2290         if (i >= usec_timeout) {
2291                 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2292                 return -EINVAL;
2293         }
2294
2295         if (amdgpu_emu_mode == 1)
2296                 adev->hdp.funcs->flush_hdp(adev, NULL);
2297
2298         tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL);
2299         tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, VMID, 0);
2300         tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, CACHE_POLICY, 0);
2301         tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, EXE_DISABLE, 0);
2302         tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, ADDRESS_CLAMP, 1);
2303         WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL, tmp);
2304
2305         /* Program pfp ucode address into intruction cache address register */
2306         WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_LO,
2307                         lower_32_bits(addr) & 0xFFFFF000);
2308         WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_HI,
2309                         upper_32_bits(addr));
2310
2311         return 0;
2312 }
2313
2314 static int gfx_v11_0_config_mec_cache(struct amdgpu_device *adev, uint64_t addr)
2315 {
2316         uint32_t usec_timeout = 50000;  /* wait for 50ms */
2317         uint32_t tmp;
2318         int i;
2319
2320         /* Trigger an invalidation of the L1 instruction caches */
2321         tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL);
2322         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_OP_CNTL, INVALIDATE_CACHE, 1);
2323
2324         WREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL, tmp);
2325
2326         /* Wait for invalidation complete */
2327         for (i = 0; i < usec_timeout; i++) {
2328                 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL);
2329                 if (1 == REG_GET_FIELD(tmp, CP_CPC_IC_OP_CNTL,
2330                                         INVALIDATE_CACHE_COMPLETE))
2331                         break;
2332                 udelay(1);
2333         }
2334
2335         if (i >= usec_timeout) {
2336                 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2337                 return -EINVAL;
2338         }
2339
2340         if (amdgpu_emu_mode == 1)
2341                 adev->hdp.funcs->flush_hdp(adev, NULL);
2342
2343         tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL);
2344         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0);
2345         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, EXE_DISABLE, 0);
2346         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, ADDRESS_CLAMP, 1);
2347         WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL, tmp);
2348
2349         /* Program mec1 ucode address into intruction cache address register */
2350         WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_LO,
2351                         lower_32_bits(addr) & 0xFFFFF000);
2352         WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_HI,
2353                         upper_32_bits(addr));
2354
2355         return 0;
2356 }
2357
2358 static int gfx_v11_0_config_pfp_cache_rs64(struct amdgpu_device *adev, uint64_t addr, uint64_t addr2)
2359 {
2360         uint32_t usec_timeout = 50000;  /* wait for 50ms */
2361         uint32_t tmp;
2362         unsigned i, pipe_id;
2363         const struct gfx_firmware_header_v2_0 *pfp_hdr;
2364
2365         pfp_hdr = (const struct gfx_firmware_header_v2_0 *)
2366                 adev->gfx.pfp_fw->data;
2367
2368         WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_LO,
2369                 lower_32_bits(addr));
2370         WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_HI,
2371                 upper_32_bits(addr));
2372
2373         tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL);
2374         tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, VMID, 0);
2375         tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, CACHE_POLICY, 0);
2376         tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, EXE_DISABLE, 0);
2377         WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL, tmp);
2378
2379         /*
2380          * Programming any of the CP_PFP_IC_BASE registers
2381          * forces invalidation of the ME L1 I$. Wait for the
2382          * invalidation complete
2383          */
2384         for (i = 0; i < usec_timeout; i++) {
2385                 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
2386                 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL,
2387                         INVALIDATE_CACHE_COMPLETE))
2388                         break;
2389                 udelay(1);
2390         }
2391
2392         if (i >= usec_timeout) {
2393                 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2394                 return -EINVAL;
2395         }
2396
2397         /* Prime the L1 instruction caches */
2398         tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
2399         tmp = REG_SET_FIELD(tmp, CP_PFP_IC_OP_CNTL, PRIME_ICACHE, 1);
2400         WREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL, tmp);
2401         /* Waiting for cache primed*/
2402         for (i = 0; i < usec_timeout; i++) {
2403                 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
2404                 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL,
2405                         ICACHE_PRIMED))
2406                         break;
2407                 udelay(1);
2408         }
2409
2410         if (i >= usec_timeout) {
2411                 dev_err(adev->dev, "failed to prime instruction cache\n");
2412                 return -EINVAL;
2413         }
2414
2415         mutex_lock(&adev->srbm_mutex);
2416         for (pipe_id = 0; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) {
2417                 soc21_grbm_select(adev, 0, pipe_id, 0, 0);
2418                 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START,
2419                         (pfp_hdr->ucode_start_addr_hi << 30) |
2420                         (pfp_hdr->ucode_start_addr_lo >> 2));
2421                 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START_HI,
2422                         pfp_hdr->ucode_start_addr_hi >> 2);
2423
2424                 /*
2425                  * Program CP_ME_CNTL to reset given PIPE to take
2426                  * effect of CP_PFP_PRGRM_CNTR_START.
2427                  */
2428                 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL);
2429                 if (pipe_id == 0)
2430                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2431                                         PFP_PIPE0_RESET, 1);
2432                 else
2433                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2434                                         PFP_PIPE1_RESET, 1);
2435                 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2436
2437                 /* Clear pfp pipe0 reset bit. */
2438                 if (pipe_id == 0)
2439                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2440                                         PFP_PIPE0_RESET, 0);
2441                 else
2442                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2443                                         PFP_PIPE1_RESET, 0);
2444                 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2445
2446                 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE0_LO,
2447                         lower_32_bits(addr2));
2448                 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE0_HI,
2449                         upper_32_bits(addr2));
2450         }
2451         soc21_grbm_select(adev, 0, 0, 0, 0);
2452         mutex_unlock(&adev->srbm_mutex);
2453
2454         tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL);
2455         tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, VMID, 0);
2456         tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, CACHE_POLICY, 0);
2457         WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL, tmp);
2458
2459         /* Invalidate the data caches */
2460         tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
2461         tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, INVALIDATE_DCACHE, 1);
2462         WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL, tmp);
2463
2464         for (i = 0; i < usec_timeout; i++) {
2465                 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
2466                 if (1 == REG_GET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL,
2467                         INVALIDATE_DCACHE_COMPLETE))
2468                         break;
2469                 udelay(1);
2470         }
2471
2472         if (i >= usec_timeout) {
2473                 dev_err(adev->dev, "failed to invalidate RS64 data cache\n");
2474                 return -EINVAL;
2475         }
2476
2477         return 0;
2478 }
2479
2480 static int gfx_v11_0_config_me_cache_rs64(struct amdgpu_device *adev, uint64_t addr, uint64_t addr2)
2481 {
2482         uint32_t usec_timeout = 50000;  /* wait for 50ms */
2483         uint32_t tmp;
2484         unsigned i, pipe_id;
2485         const struct gfx_firmware_header_v2_0 *me_hdr;
2486
2487         me_hdr = (const struct gfx_firmware_header_v2_0 *)
2488                 adev->gfx.me_fw->data;
2489
2490         WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_LO,
2491                 lower_32_bits(addr));
2492         WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_HI,
2493                 upper_32_bits(addr));
2494
2495         tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL);
2496         tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, VMID, 0);
2497         tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, CACHE_POLICY, 0);
2498         tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, EXE_DISABLE, 0);
2499         WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL, tmp);
2500
2501         /*
2502          * Programming any of the CP_ME_IC_BASE registers
2503          * forces invalidation of the ME L1 I$. Wait for the
2504          * invalidation complete
2505          */
2506         for (i = 0; i < usec_timeout; i++) {
2507                 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
2508                 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL,
2509                         INVALIDATE_CACHE_COMPLETE))
2510                         break;
2511                 udelay(1);
2512         }
2513
2514         if (i >= usec_timeout) {
2515                 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2516                 return -EINVAL;
2517         }
2518
2519         /* Prime the instruction caches */
2520         tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
2521         tmp = REG_SET_FIELD(tmp, CP_ME_IC_OP_CNTL, PRIME_ICACHE, 1);
2522         WREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL, tmp);
2523
2524         /* Waiting for instruction cache primed*/
2525         for (i = 0; i < usec_timeout; i++) {
2526                 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
2527                 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL,
2528                         ICACHE_PRIMED))
2529                         break;
2530                 udelay(1);
2531         }
2532
2533         if (i >= usec_timeout) {
2534                 dev_err(adev->dev, "failed to prime instruction cache\n");
2535                 return -EINVAL;
2536         }
2537
2538         mutex_lock(&adev->srbm_mutex);
2539         for (pipe_id = 0; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) {
2540                 soc21_grbm_select(adev, 0, pipe_id, 0, 0);
2541                 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START,
2542                         (me_hdr->ucode_start_addr_hi << 30) |
2543                         (me_hdr->ucode_start_addr_lo >> 2) );
2544                 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START_HI,
2545                         me_hdr->ucode_start_addr_hi>>2);
2546
2547                 /*
2548                  * Program CP_ME_CNTL to reset given PIPE to take
2549                  * effect of CP_PFP_PRGRM_CNTR_START.
2550                  */
2551                 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL);
2552                 if (pipe_id == 0)
2553                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2554                                         ME_PIPE0_RESET, 1);
2555                 else
2556                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2557                                         ME_PIPE1_RESET, 1);
2558                 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2559
2560                 /* Clear pfp pipe0 reset bit. */
2561                 if (pipe_id == 0)
2562                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2563                                         ME_PIPE0_RESET, 0);
2564                 else
2565                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2566                                         ME_PIPE1_RESET, 0);
2567                 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2568
2569                 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE1_LO,
2570                         lower_32_bits(addr2));
2571                 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE1_HI,
2572                         upper_32_bits(addr2));
2573         }
2574         soc21_grbm_select(adev, 0, 0, 0, 0);
2575         mutex_unlock(&adev->srbm_mutex);
2576
2577         tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL);
2578         tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, VMID, 0);
2579         tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, CACHE_POLICY, 0);
2580         WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL, tmp);
2581
2582         /* Invalidate the data caches */
2583         tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
2584         tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, INVALIDATE_DCACHE, 1);
2585         WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL, tmp);
2586
2587         for (i = 0; i < usec_timeout; i++) {
2588                 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
2589                 if (1 == REG_GET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL,
2590                         INVALIDATE_DCACHE_COMPLETE))
2591                         break;
2592                 udelay(1);
2593         }
2594
2595         if (i >= usec_timeout) {
2596                 dev_err(adev->dev, "failed to invalidate RS64 data cache\n");
2597                 return -EINVAL;
2598         }
2599
2600         return 0;
2601 }
2602
2603 static int gfx_v11_0_config_mec_cache_rs64(struct amdgpu_device *adev, uint64_t addr, uint64_t addr2)
2604 {
2605         uint32_t usec_timeout = 50000;  /* wait for 50ms */
2606         uint32_t tmp;
2607         unsigned i;
2608         const struct gfx_firmware_header_v2_0 *mec_hdr;
2609
2610         mec_hdr = (const struct gfx_firmware_header_v2_0 *)
2611                 adev->gfx.mec_fw->data;
2612
2613         tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL);
2614         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0);
2615         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, EXE_DISABLE, 0);
2616         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0);
2617         WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL, tmp);
2618
2619         tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_BASE_CNTL);
2620         tmp = REG_SET_FIELD(tmp, CP_MEC_DC_BASE_CNTL, VMID, 0);
2621         tmp = REG_SET_FIELD(tmp, CP_MEC_DC_BASE_CNTL, CACHE_POLICY, 0);
2622         WREG32_SOC15(GC, 0, regCP_MEC_DC_BASE_CNTL, tmp);
2623
2624         mutex_lock(&adev->srbm_mutex);
2625         for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) {
2626                 soc21_grbm_select(adev, 1, i, 0, 0);
2627
2628                 WREG32_SOC15(GC, 0, regCP_MEC_MDBASE_LO, addr2);
2629                 WREG32_SOC15(GC, 0, regCP_MEC_MDBASE_HI,
2630                      upper_32_bits(addr2));
2631
2632                 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START,
2633                                         mec_hdr->ucode_start_addr_lo >> 2 |
2634                                         mec_hdr->ucode_start_addr_hi << 30);
2635                 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START_HI,
2636                                         mec_hdr->ucode_start_addr_hi >> 2);
2637
2638                 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_LO, addr);
2639                 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_HI,
2640                      upper_32_bits(addr));
2641         }
2642         mutex_unlock(&adev->srbm_mutex);
2643         soc21_grbm_select(adev, 0, 0, 0, 0);
2644
2645         /* Trigger an invalidation of the L1 instruction caches */
2646         tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL);
2647         tmp = REG_SET_FIELD(tmp, CP_MEC_DC_OP_CNTL, INVALIDATE_DCACHE, 1);
2648         WREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL, tmp);
2649
2650         /* Wait for invalidation complete */
2651         for (i = 0; i < usec_timeout; i++) {
2652                 tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL);
2653                 if (1 == REG_GET_FIELD(tmp, CP_MEC_DC_OP_CNTL,
2654                                        INVALIDATE_DCACHE_COMPLETE))
2655                         break;
2656                 udelay(1);
2657         }
2658
2659         if (i >= usec_timeout) {
2660                 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2661                 return -EINVAL;
2662         }
2663
2664         /* Trigger an invalidation of the L1 instruction caches */
2665         tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL);
2666         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_OP_CNTL, INVALIDATE_CACHE, 1);
2667         WREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL, tmp);
2668
2669         /* Wait for invalidation complete */
2670         for (i = 0; i < usec_timeout; i++) {
2671                 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL);
2672                 if (1 == REG_GET_FIELD(tmp, CP_CPC_IC_OP_CNTL,
2673                                        INVALIDATE_CACHE_COMPLETE))
2674                         break;
2675                 udelay(1);
2676         }
2677
2678         if (i >= usec_timeout) {
2679                 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2680                 return -EINVAL;
2681         }
2682
2683         return 0;
2684 }
2685
2686 static void gfx_v11_0_config_gfx_rs64(struct amdgpu_device *adev)
2687 {
2688         const struct gfx_firmware_header_v2_0 *pfp_hdr;
2689         const struct gfx_firmware_header_v2_0 *me_hdr;
2690         const struct gfx_firmware_header_v2_0 *mec_hdr;
2691         uint32_t pipe_id, tmp;
2692
2693         mec_hdr = (const struct gfx_firmware_header_v2_0 *)
2694                 adev->gfx.mec_fw->data;
2695         me_hdr = (const struct gfx_firmware_header_v2_0 *)
2696                 adev->gfx.me_fw->data;
2697         pfp_hdr = (const struct gfx_firmware_header_v2_0 *)
2698                 adev->gfx.pfp_fw->data;
2699
2700         /* config pfp program start addr */
2701         for (pipe_id = 0; pipe_id < 2; pipe_id++) {
2702                 soc21_grbm_select(adev, 0, pipe_id, 0, 0);
2703                 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START,
2704                         (pfp_hdr->ucode_start_addr_hi << 30) |
2705                         (pfp_hdr->ucode_start_addr_lo >> 2));
2706                 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START_HI,
2707                         pfp_hdr->ucode_start_addr_hi >> 2);
2708         }
2709         soc21_grbm_select(adev, 0, 0, 0, 0);
2710
2711         /* reset pfp pipe */
2712         tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL);
2713         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE0_RESET, 1);
2714         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE1_RESET, 1);
2715         WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2716
2717         /* clear pfp pipe reset */
2718         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE0_RESET, 0);
2719         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE1_RESET, 0);
2720         WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2721
2722         /* config me program start addr */
2723         for (pipe_id = 0; pipe_id < 2; pipe_id++) {
2724                 soc21_grbm_select(adev, 0, pipe_id, 0, 0);
2725                 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START,
2726                         (me_hdr->ucode_start_addr_hi << 30) |
2727                         (me_hdr->ucode_start_addr_lo >> 2) );
2728                 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START_HI,
2729                         me_hdr->ucode_start_addr_hi>>2);
2730         }
2731         soc21_grbm_select(adev, 0, 0, 0, 0);
2732
2733         /* reset me pipe */
2734         tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL);
2735         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE0_RESET, 1);
2736         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE1_RESET, 1);
2737         WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2738
2739         /* clear me pipe reset */
2740         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE0_RESET, 0);
2741         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE1_RESET, 0);
2742         WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2743
2744         /* config mec program start addr */
2745         for (pipe_id = 0; pipe_id < 4; pipe_id++) {
2746                 soc21_grbm_select(adev, 1, pipe_id, 0, 0);
2747                 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START,
2748                                         mec_hdr->ucode_start_addr_lo >> 2 |
2749                                         mec_hdr->ucode_start_addr_hi << 30);
2750                 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START_HI,
2751                                         mec_hdr->ucode_start_addr_hi >> 2);
2752         }
2753         soc21_grbm_select(adev, 0, 0, 0, 0);
2754 }
2755
2756 static int gfx_v11_0_wait_for_rlc_autoload_complete(struct amdgpu_device *adev)
2757 {
2758         uint32_t cp_status;
2759         uint32_t bootload_status;
2760         int i, r;
2761         uint64_t addr, addr2;
2762
2763         for (i = 0; i < adev->usec_timeout; i++) {
2764                 cp_status = RREG32_SOC15(GC, 0, regCP_STAT);
2765                 bootload_status = RREG32_SOC15(GC, 0, regRLC_RLCS_BOOTLOAD_STATUS);
2766                 if ((cp_status == 0) &&
2767                     (REG_GET_FIELD(bootload_status,
2768                         RLC_RLCS_BOOTLOAD_STATUS, BOOTLOAD_COMPLETE) == 1)) {
2769                         break;
2770                 }
2771                 udelay(1);
2772         }
2773
2774         if (i >= adev->usec_timeout) {
2775                 dev_err(adev->dev, "rlc autoload: gc ucode autoload timeout\n");
2776                 return -ETIMEDOUT;
2777         }
2778
2779         if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) {
2780                 if (adev->gfx.rs64_enable) {
2781                         addr = adev->gfx.rlc.rlc_autoload_gpu_addr +
2782                                 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_ME].offset;
2783                         addr2 = adev->gfx.rlc.rlc_autoload_gpu_addr +
2784                                 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_ME_P0_STACK].offset;
2785                         r = gfx_v11_0_config_me_cache_rs64(adev, addr, addr2);
2786                         if (r)
2787                                 return r;
2788                         addr = adev->gfx.rlc.rlc_autoload_gpu_addr +
2789                                 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_PFP].offset;
2790                         addr2 = adev->gfx.rlc.rlc_autoload_gpu_addr +
2791                                 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_PFP_P0_STACK].offset;
2792                         r = gfx_v11_0_config_pfp_cache_rs64(adev, addr, addr2);
2793                         if (r)
2794                                 return r;
2795                         addr = adev->gfx.rlc.rlc_autoload_gpu_addr +
2796                                 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_MEC].offset;
2797                         addr2 = adev->gfx.rlc.rlc_autoload_gpu_addr +
2798                                 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_MEC_P0_STACK].offset;
2799                         r = gfx_v11_0_config_mec_cache_rs64(adev, addr, addr2);
2800                         if (r)
2801                                 return r;
2802                 } else {
2803                         addr = adev->gfx.rlc.rlc_autoload_gpu_addr +
2804                                 rlc_autoload_info[SOC21_FIRMWARE_ID_CP_ME].offset;
2805                         r = gfx_v11_0_config_me_cache(adev, addr);
2806                         if (r)
2807                                 return r;
2808                         addr = adev->gfx.rlc.rlc_autoload_gpu_addr +
2809                                 rlc_autoload_info[SOC21_FIRMWARE_ID_CP_PFP].offset;
2810                         r = gfx_v11_0_config_pfp_cache(adev, addr);
2811                         if (r)
2812                                 return r;
2813                         addr = adev->gfx.rlc.rlc_autoload_gpu_addr +
2814                                 rlc_autoload_info[SOC21_FIRMWARE_ID_CP_MEC].offset;
2815                         r = gfx_v11_0_config_mec_cache(adev, addr);
2816                         if (r)
2817                                 return r;
2818                 }
2819         }
2820
2821         return 0;
2822 }
2823
2824 static int gfx_v11_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable)
2825 {
2826         int i;
2827         u32 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL);
2828
2829         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, enable ? 0 : 1);
2830         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, enable ? 0 : 1);
2831         WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2832
2833         for (i = 0; i < adev->usec_timeout; i++) {
2834                 if (RREG32_SOC15(GC, 0, regCP_STAT) == 0)
2835                         break;
2836                 udelay(1);
2837         }
2838
2839         if (i >= adev->usec_timeout)
2840                 DRM_ERROR("failed to %s cp gfx\n", enable ? "unhalt" : "halt");
2841
2842         return 0;
2843 }
2844
2845 static int gfx_v11_0_cp_gfx_load_pfp_microcode(struct amdgpu_device *adev)
2846 {
2847         int r;
2848         const struct gfx_firmware_header_v1_0 *pfp_hdr;
2849         const __le32 *fw_data;
2850         unsigned i, fw_size;
2851
2852         pfp_hdr = (const struct gfx_firmware_header_v1_0 *)
2853                 adev->gfx.pfp_fw->data;
2854
2855         amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header);
2856
2857         fw_data = (const __le32 *)(adev->gfx.pfp_fw->data +
2858                 le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
2859         fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes);
2860
2861         r = amdgpu_bo_create_reserved(adev, pfp_hdr->header.ucode_size_bytes,
2862                                       PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
2863                                       &adev->gfx.pfp.pfp_fw_obj,
2864                                       &adev->gfx.pfp.pfp_fw_gpu_addr,
2865                                       (void **)&adev->gfx.pfp.pfp_fw_ptr);
2866         if (r) {
2867                 dev_err(adev->dev, "(%d) failed to create pfp fw bo\n", r);
2868                 gfx_v11_0_pfp_fini(adev);
2869                 return r;
2870         }
2871
2872         memcpy(adev->gfx.pfp.pfp_fw_ptr, fw_data, fw_size);
2873
2874         amdgpu_bo_kunmap(adev->gfx.pfp.pfp_fw_obj);
2875         amdgpu_bo_unreserve(adev->gfx.pfp.pfp_fw_obj);
2876
2877         gfx_v11_0_config_pfp_cache(adev, adev->gfx.pfp.pfp_fw_gpu_addr);
2878
2879         WREG32_SOC15(GC, 0, regCP_HYP_PFP_UCODE_ADDR, 0);
2880
2881         for (i = 0; i < pfp_hdr->jt_size; i++)
2882                 WREG32_SOC15(GC, 0, regCP_HYP_PFP_UCODE_DATA,
2883                              le32_to_cpup(fw_data + pfp_hdr->jt_offset + i));
2884
2885         WREG32_SOC15(GC, 0, regCP_HYP_PFP_UCODE_ADDR, adev->gfx.pfp_fw_version);
2886
2887         return 0;
2888 }
2889
2890 static int gfx_v11_0_cp_gfx_load_pfp_microcode_rs64(struct amdgpu_device *adev)
2891 {
2892         int r;
2893         const struct gfx_firmware_header_v2_0 *pfp_hdr;
2894         const __le32 *fw_ucode, *fw_data;
2895         unsigned i, pipe_id, fw_ucode_size, fw_data_size;
2896         uint32_t tmp;
2897         uint32_t usec_timeout = 50000;  /* wait for 50ms */
2898
2899         pfp_hdr = (const struct gfx_firmware_header_v2_0 *)
2900                 adev->gfx.pfp_fw->data;
2901
2902         amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header);
2903
2904         /* instruction */
2905         fw_ucode = (const __le32 *)(adev->gfx.pfp_fw->data +
2906                 le32_to_cpu(pfp_hdr->ucode_offset_bytes));
2907         fw_ucode_size = le32_to_cpu(pfp_hdr->ucode_size_bytes);
2908         /* data */
2909         fw_data = (const __le32 *)(adev->gfx.pfp_fw->data +
2910                 le32_to_cpu(pfp_hdr->data_offset_bytes));
2911         fw_data_size = le32_to_cpu(pfp_hdr->data_size_bytes);
2912
2913         /* 64kb align */
2914         r = amdgpu_bo_create_reserved(adev, fw_ucode_size,
2915                                       64 * 1024, AMDGPU_GEM_DOMAIN_VRAM,
2916                                       &adev->gfx.pfp.pfp_fw_obj,
2917                                       &adev->gfx.pfp.pfp_fw_gpu_addr,
2918                                       (void **)&adev->gfx.pfp.pfp_fw_ptr);
2919         if (r) {
2920                 dev_err(adev->dev, "(%d) failed to create pfp ucode fw bo\n", r);
2921                 gfx_v11_0_pfp_fini(adev);
2922                 return r;
2923         }
2924
2925         r = amdgpu_bo_create_reserved(adev, fw_data_size,
2926                                       64 * 1024, AMDGPU_GEM_DOMAIN_VRAM,
2927                                       &adev->gfx.pfp.pfp_fw_data_obj,
2928                                       &adev->gfx.pfp.pfp_fw_data_gpu_addr,
2929                                       (void **)&adev->gfx.pfp.pfp_fw_data_ptr);
2930         if (r) {
2931                 dev_err(adev->dev, "(%d) failed to create pfp data fw bo\n", r);
2932                 gfx_v11_0_pfp_fini(adev);
2933                 return r;
2934         }
2935
2936         memcpy(adev->gfx.pfp.pfp_fw_ptr, fw_ucode, fw_ucode_size);
2937         memcpy(adev->gfx.pfp.pfp_fw_data_ptr, fw_data, fw_data_size);
2938
2939         amdgpu_bo_kunmap(adev->gfx.pfp.pfp_fw_obj);
2940         amdgpu_bo_kunmap(adev->gfx.pfp.pfp_fw_data_obj);
2941         amdgpu_bo_unreserve(adev->gfx.pfp.pfp_fw_obj);
2942         amdgpu_bo_unreserve(adev->gfx.pfp.pfp_fw_data_obj);
2943
2944         if (amdgpu_emu_mode == 1)
2945                 adev->hdp.funcs->flush_hdp(adev, NULL);
2946
2947         WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_LO,
2948                 lower_32_bits(adev->gfx.pfp.pfp_fw_gpu_addr));
2949         WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_HI,
2950                 upper_32_bits(adev->gfx.pfp.pfp_fw_gpu_addr));
2951
2952         tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL);
2953         tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, VMID, 0);
2954         tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, CACHE_POLICY, 0);
2955         tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, EXE_DISABLE, 0);
2956         WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL, tmp);
2957
2958         /*
2959          * Programming any of the CP_PFP_IC_BASE registers
2960          * forces invalidation of the ME L1 I$. Wait for the
2961          * invalidation complete
2962          */
2963         for (i = 0; i < usec_timeout; i++) {
2964                 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
2965                 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL,
2966                         INVALIDATE_CACHE_COMPLETE))
2967                         break;
2968                 udelay(1);
2969         }
2970
2971         if (i >= usec_timeout) {
2972                 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2973                 return -EINVAL;
2974         }
2975
2976         /* Prime the L1 instruction caches */
2977         tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
2978         tmp = REG_SET_FIELD(tmp, CP_PFP_IC_OP_CNTL, PRIME_ICACHE, 1);
2979         WREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL, tmp);
2980         /* Waiting for cache primed*/
2981         for (i = 0; i < usec_timeout; i++) {
2982                 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
2983                 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL,
2984                         ICACHE_PRIMED))
2985                         break;
2986                 udelay(1);
2987         }
2988
2989         if (i >= usec_timeout) {
2990                 dev_err(adev->dev, "failed to prime instruction cache\n");
2991                 return -EINVAL;
2992         }
2993
2994         mutex_lock(&adev->srbm_mutex);
2995         for (pipe_id = 0; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) {
2996                 soc21_grbm_select(adev, 0, pipe_id, 0, 0);
2997                 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START,
2998                         (pfp_hdr->ucode_start_addr_hi << 30) |
2999                         (pfp_hdr->ucode_start_addr_lo >> 2) );
3000                 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START_HI,
3001                         pfp_hdr->ucode_start_addr_hi>>2);
3002
3003                 /*
3004                  * Program CP_ME_CNTL to reset given PIPE to take
3005                  * effect of CP_PFP_PRGRM_CNTR_START.
3006                  */
3007                 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL);
3008                 if (pipe_id == 0)
3009                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
3010                                         PFP_PIPE0_RESET, 1);
3011                 else
3012                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
3013                                         PFP_PIPE1_RESET, 1);
3014                 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
3015
3016                 /* Clear pfp pipe0 reset bit. */
3017                 if (pipe_id == 0)
3018                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
3019                                         PFP_PIPE0_RESET, 0);
3020                 else
3021                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
3022                                         PFP_PIPE1_RESET, 0);
3023                 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
3024
3025                 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE0_LO,
3026                         lower_32_bits(adev->gfx.pfp.pfp_fw_data_gpu_addr));
3027                 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE0_HI,
3028                         upper_32_bits(adev->gfx.pfp.pfp_fw_data_gpu_addr));
3029         }
3030         soc21_grbm_select(adev, 0, 0, 0, 0);
3031         mutex_unlock(&adev->srbm_mutex);
3032
3033         tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL);
3034         tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, VMID, 0);
3035         tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, CACHE_POLICY, 0);
3036         WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL, tmp);
3037
3038         /* Invalidate the data caches */
3039         tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
3040         tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, INVALIDATE_DCACHE, 1);
3041         WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL, tmp);
3042
3043         for (i = 0; i < usec_timeout; i++) {
3044                 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
3045                 if (1 == REG_GET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL,
3046                         INVALIDATE_DCACHE_COMPLETE))
3047                         break;
3048                 udelay(1);
3049         }
3050
3051         if (i >= usec_timeout) {
3052                 dev_err(adev->dev, "failed to invalidate RS64 data cache\n");
3053                 return -EINVAL;
3054         }
3055
3056         return 0;
3057 }
3058
3059 static int gfx_v11_0_cp_gfx_load_me_microcode(struct amdgpu_device *adev)
3060 {
3061         int r;
3062         const struct gfx_firmware_header_v1_0 *me_hdr;
3063         const __le32 *fw_data;
3064         unsigned i, fw_size;
3065
3066         me_hdr = (const struct gfx_firmware_header_v1_0 *)
3067                 adev->gfx.me_fw->data;
3068
3069         amdgpu_ucode_print_gfx_hdr(&me_hdr->header);
3070
3071         fw_data = (const __le32 *)(adev->gfx.me_fw->data +
3072                 le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
3073         fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes);
3074
3075         r = amdgpu_bo_create_reserved(adev, me_hdr->header.ucode_size_bytes,
3076                                       PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
3077                                       &adev->gfx.me.me_fw_obj,
3078                                       &adev->gfx.me.me_fw_gpu_addr,
3079                                       (void **)&adev->gfx.me.me_fw_ptr);
3080         if (r) {
3081                 dev_err(adev->dev, "(%d) failed to create me fw bo\n", r);
3082                 gfx_v11_0_me_fini(adev);
3083                 return r;
3084         }
3085
3086         memcpy(adev->gfx.me.me_fw_ptr, fw_data, fw_size);
3087
3088         amdgpu_bo_kunmap(adev->gfx.me.me_fw_obj);
3089         amdgpu_bo_unreserve(adev->gfx.me.me_fw_obj);
3090
3091         gfx_v11_0_config_me_cache(adev, adev->gfx.me.me_fw_gpu_addr);
3092
3093         WREG32_SOC15(GC, 0, regCP_HYP_ME_UCODE_ADDR, 0);
3094
3095         for (i = 0; i < me_hdr->jt_size; i++)
3096                 WREG32_SOC15(GC, 0, regCP_HYP_ME_UCODE_DATA,
3097                              le32_to_cpup(fw_data + me_hdr->jt_offset + i));
3098
3099         WREG32_SOC15(GC, 0, regCP_HYP_ME_UCODE_ADDR, adev->gfx.me_fw_version);
3100
3101         return 0;
3102 }
3103
3104 static int gfx_v11_0_cp_gfx_load_me_microcode_rs64(struct amdgpu_device *adev)
3105 {
3106         int r;
3107         const struct gfx_firmware_header_v2_0 *me_hdr;
3108         const __le32 *fw_ucode, *fw_data;
3109         unsigned i, pipe_id, fw_ucode_size, fw_data_size;
3110         uint32_t tmp;
3111         uint32_t usec_timeout = 50000;  /* wait for 50ms */
3112
3113         me_hdr = (const struct gfx_firmware_header_v2_0 *)
3114                 adev->gfx.me_fw->data;
3115
3116         amdgpu_ucode_print_gfx_hdr(&me_hdr->header);
3117
3118         /* instruction */
3119         fw_ucode = (const __le32 *)(adev->gfx.me_fw->data +
3120                 le32_to_cpu(me_hdr->ucode_offset_bytes));
3121         fw_ucode_size = le32_to_cpu(me_hdr->ucode_size_bytes);
3122         /* data */
3123         fw_data = (const __le32 *)(adev->gfx.me_fw->data +
3124                 le32_to_cpu(me_hdr->data_offset_bytes));
3125         fw_data_size = le32_to_cpu(me_hdr->data_size_bytes);
3126
3127         /* 64kb align*/
3128         r = amdgpu_bo_create_reserved(adev, fw_ucode_size,
3129                                       64 * 1024, AMDGPU_GEM_DOMAIN_VRAM,
3130                                       &adev->gfx.me.me_fw_obj,
3131                                       &adev->gfx.me.me_fw_gpu_addr,
3132                                       (void **)&adev->gfx.me.me_fw_ptr);
3133         if (r) {
3134                 dev_err(adev->dev, "(%d) failed to create me ucode bo\n", r);
3135                 gfx_v11_0_me_fini(adev);
3136                 return r;
3137         }
3138
3139         r = amdgpu_bo_create_reserved(adev, fw_data_size,
3140                                       64 * 1024, AMDGPU_GEM_DOMAIN_VRAM,
3141                                       &adev->gfx.me.me_fw_data_obj,
3142                                       &adev->gfx.me.me_fw_data_gpu_addr,
3143                                       (void **)&adev->gfx.me.me_fw_data_ptr);
3144         if (r) {
3145                 dev_err(adev->dev, "(%d) failed to create me data bo\n", r);
3146                 gfx_v11_0_pfp_fini(adev);
3147                 return r;
3148         }
3149
3150         memcpy(adev->gfx.me.me_fw_ptr, fw_ucode, fw_ucode_size);
3151         memcpy(adev->gfx.me.me_fw_data_ptr, fw_data, fw_data_size);
3152
3153         amdgpu_bo_kunmap(adev->gfx.me.me_fw_obj);
3154         amdgpu_bo_kunmap(adev->gfx.me.me_fw_data_obj);
3155         amdgpu_bo_unreserve(adev->gfx.me.me_fw_obj);
3156         amdgpu_bo_unreserve(adev->gfx.me.me_fw_data_obj);
3157
3158         if (amdgpu_emu_mode == 1)
3159                 adev->hdp.funcs->flush_hdp(adev, NULL);
3160
3161         WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_LO,
3162                 lower_32_bits(adev->gfx.me.me_fw_gpu_addr));
3163         WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_HI,
3164                 upper_32_bits(adev->gfx.me.me_fw_gpu_addr));
3165
3166         tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL);
3167         tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, VMID, 0);
3168         tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, CACHE_POLICY, 0);
3169         tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, EXE_DISABLE, 0);
3170         WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL, tmp);
3171
3172         /*
3173          * Programming any of the CP_ME_IC_BASE registers
3174          * forces invalidation of the ME L1 I$. Wait for the
3175          * invalidation complete
3176          */
3177         for (i = 0; i < usec_timeout; i++) {
3178                 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
3179                 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL,
3180                         INVALIDATE_CACHE_COMPLETE))
3181                         break;
3182                 udelay(1);
3183         }
3184
3185         if (i >= usec_timeout) {
3186                 dev_err(adev->dev, "failed to invalidate instruction cache\n");
3187                 return -EINVAL;
3188         }
3189
3190         /* Prime the instruction caches */
3191         tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
3192         tmp = REG_SET_FIELD(tmp, CP_ME_IC_OP_CNTL, PRIME_ICACHE, 1);
3193         WREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL, tmp);
3194
3195         /* Waiting for instruction cache primed*/
3196         for (i = 0; i < usec_timeout; i++) {
3197                 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
3198                 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL,
3199                         ICACHE_PRIMED))
3200                         break;
3201                 udelay(1);
3202         }
3203
3204         if (i >= usec_timeout) {
3205                 dev_err(adev->dev, "failed to prime instruction cache\n");
3206                 return -EINVAL;
3207         }
3208
3209         mutex_lock(&adev->srbm_mutex);
3210         for (pipe_id = 0; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) {
3211                 soc21_grbm_select(adev, 0, pipe_id, 0, 0);
3212                 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START,
3213                         (me_hdr->ucode_start_addr_hi << 30) |
3214                         (me_hdr->ucode_start_addr_lo >> 2) );
3215                 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START_HI,
3216                         me_hdr->ucode_start_addr_hi>>2);
3217
3218                 /*
3219                  * Program CP_ME_CNTL to reset given PIPE to take
3220                  * effect of CP_PFP_PRGRM_CNTR_START.
3221                  */
3222                 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL);
3223                 if (pipe_id == 0)
3224                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
3225                                         ME_PIPE0_RESET, 1);
3226                 else
3227                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
3228                                         ME_PIPE1_RESET, 1);
3229                 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
3230
3231                 /* Clear pfp pipe0 reset bit. */
3232                 if (pipe_id == 0)
3233                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
3234                                         ME_PIPE0_RESET, 0);
3235                 else
3236                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
3237                                         ME_PIPE1_RESET, 0);
3238                 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
3239
3240                 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE1_LO,
3241                         lower_32_bits(adev->gfx.me.me_fw_data_gpu_addr));
3242                 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE1_HI,
3243                         upper_32_bits(adev->gfx.me.me_fw_data_gpu_addr));
3244         }
3245         soc21_grbm_select(adev, 0, 0, 0, 0);
3246         mutex_unlock(&adev->srbm_mutex);
3247
3248         tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL);
3249         tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, VMID, 0);
3250         tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, CACHE_POLICY, 0);
3251         WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL, tmp);
3252
3253         /* Invalidate the data caches */
3254         tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
3255         tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, INVALIDATE_DCACHE, 1);
3256         WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL, tmp);
3257
3258         for (i = 0; i < usec_timeout; i++) {
3259                 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
3260                 if (1 == REG_GET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL,
3261                         INVALIDATE_DCACHE_COMPLETE))
3262                         break;
3263                 udelay(1);
3264         }
3265
3266         if (i >= usec_timeout) {
3267                 dev_err(adev->dev, "failed to invalidate RS64 data cache\n");
3268                 return -EINVAL;
3269         }
3270
3271         return 0;
3272 }
3273
3274 static int gfx_v11_0_cp_gfx_load_microcode(struct amdgpu_device *adev)
3275 {
3276         int r;
3277
3278         if (!adev->gfx.me_fw || !adev->gfx.pfp_fw)
3279                 return -EINVAL;
3280
3281         gfx_v11_0_cp_gfx_enable(adev, false);
3282
3283         if (adev->gfx.rs64_enable)
3284                 r = gfx_v11_0_cp_gfx_load_pfp_microcode_rs64(adev);
3285         else
3286                 r = gfx_v11_0_cp_gfx_load_pfp_microcode(adev);
3287         if (r) {
3288                 dev_err(adev->dev, "(%d) failed to load pfp fw\n", r);
3289                 return r;
3290         }
3291
3292         if (adev->gfx.rs64_enable)
3293                 r = gfx_v11_0_cp_gfx_load_me_microcode_rs64(adev);
3294         else
3295                 r = gfx_v11_0_cp_gfx_load_me_microcode(adev);
3296         if (r) {
3297                 dev_err(adev->dev, "(%d) failed to load me fw\n", r);
3298                 return r;
3299         }
3300
3301         return 0;
3302 }
3303
3304 static int gfx_v11_0_cp_gfx_start(struct amdgpu_device *adev)
3305 {
3306         struct amdgpu_ring *ring;
3307         const struct cs_section_def *sect = NULL;
3308         const struct cs_extent_def *ext = NULL;
3309         int r, i;
3310         int ctx_reg_offset;
3311
3312         /* init the CP */
3313         WREG32_SOC15(GC, 0, regCP_MAX_CONTEXT,
3314                      adev->gfx.config.max_hw_contexts - 1);
3315         WREG32_SOC15(GC, 0, regCP_DEVICE_ID, 1);
3316
3317         if (!amdgpu_async_gfx_ring)
3318                 gfx_v11_0_cp_gfx_enable(adev, true);
3319
3320         ring = &adev->gfx.gfx_ring[0];
3321         r = amdgpu_ring_alloc(ring, gfx_v11_0_get_csb_size(adev));
3322         if (r) {
3323                 DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r);
3324                 return r;
3325         }
3326
3327         amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3328         amdgpu_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3329
3330         amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
3331         amdgpu_ring_write(ring, 0x80000000);
3332         amdgpu_ring_write(ring, 0x80000000);
3333
3334         for (sect = gfx11_cs_data; sect->section != NULL; ++sect) {
3335                 for (ext = sect->section; ext->extent != NULL; ++ext) {
3336                         if (sect->id == SECT_CONTEXT) {
3337                                 amdgpu_ring_write(ring,
3338                                                   PACKET3(PACKET3_SET_CONTEXT_REG,
3339                                                           ext->reg_count));
3340                                 amdgpu_ring_write(ring, ext->reg_index -
3341                                                   PACKET3_SET_CONTEXT_REG_START);
3342                                 for (i = 0; i < ext->reg_count; i++)
3343                                         amdgpu_ring_write(ring, ext->extent[i]);
3344                         }
3345                 }
3346         }
3347
3348         ctx_reg_offset =
3349                 SOC15_REG_OFFSET(GC, 0, regPA_SC_TILE_STEERING_OVERRIDE) - PACKET3_SET_CONTEXT_REG_START;
3350         amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 1));
3351         amdgpu_ring_write(ring, ctx_reg_offset);
3352         amdgpu_ring_write(ring, adev->gfx.config.pa_sc_tile_steering_override);
3353
3354         amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3355         amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3356
3357         amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3358         amdgpu_ring_write(ring, 0);
3359
3360         amdgpu_ring_commit(ring);
3361
3362         /* submit cs packet to copy state 0 to next available state */
3363         if (adev->gfx.num_gfx_rings > 1) {
3364                 /* maximum supported gfx ring is 2 */
3365                 ring = &adev->gfx.gfx_ring[1];
3366                 r = amdgpu_ring_alloc(ring, 2);
3367                 if (r) {
3368                         DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r);
3369                         return r;
3370                 }
3371
3372                 amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3373                 amdgpu_ring_write(ring, 0);
3374
3375                 amdgpu_ring_commit(ring);
3376         }
3377         return 0;
3378 }
3379
3380 static void gfx_v11_0_cp_gfx_switch_pipe(struct amdgpu_device *adev,
3381                                          CP_PIPE_ID pipe)
3382 {
3383         u32 tmp;
3384
3385         tmp = RREG32_SOC15(GC, 0, regGRBM_GFX_CNTL);
3386         tmp = REG_SET_FIELD(tmp, GRBM_GFX_CNTL, PIPEID, pipe);
3387
3388         WREG32_SOC15(GC, 0, regGRBM_GFX_CNTL, tmp);
3389 }
3390
3391 static void gfx_v11_0_cp_gfx_set_doorbell(struct amdgpu_device *adev,
3392                                           struct amdgpu_ring *ring)
3393 {
3394         u32 tmp;
3395
3396         tmp = RREG32_SOC15(GC, 0, regCP_RB_DOORBELL_CONTROL);
3397         if (ring->use_doorbell) {
3398                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3399                                     DOORBELL_OFFSET, ring->doorbell_index);
3400                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3401                                     DOORBELL_EN, 1);
3402         } else {
3403                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3404                                     DOORBELL_EN, 0);
3405         }
3406         WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_CONTROL, tmp);
3407
3408         tmp = REG_SET_FIELD(0, CP_RB_DOORBELL_RANGE_LOWER,
3409                             DOORBELL_RANGE_LOWER, ring->doorbell_index);
3410         WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_RANGE_LOWER, tmp);
3411
3412         WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_RANGE_UPPER,
3413                      CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK);
3414 }
3415
3416 static int gfx_v11_0_cp_gfx_resume(struct amdgpu_device *adev)
3417 {
3418         struct amdgpu_ring *ring;
3419         u32 tmp;
3420         u32 rb_bufsz;
3421         u64 rb_addr, rptr_addr, wptr_gpu_addr;
3422         u32 i;
3423
3424         /* Set the write pointer delay */
3425         WREG32_SOC15(GC, 0, regCP_RB_WPTR_DELAY, 0);
3426
3427         /* set the RB to use vmid 0 */
3428         WREG32_SOC15(GC, 0, regCP_RB_VMID, 0);
3429
3430         /* Init gfx ring 0 for pipe 0 */
3431         mutex_lock(&adev->srbm_mutex);
3432         gfx_v11_0_cp_gfx_switch_pipe(adev, PIPE_ID0);
3433
3434         /* Set ring buffer size */
3435         ring = &adev->gfx.gfx_ring[0];
3436         rb_bufsz = order_base_2(ring->ring_size / 8);
3437         tmp = REG_SET_FIELD(0, CP_RB0_CNTL, RB_BUFSZ, rb_bufsz);
3438         tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, RB_BLKSZ, rb_bufsz - 2);
3439         WREG32_SOC15(GC, 0, regCP_RB0_CNTL, tmp);
3440
3441         /* Initialize the ring buffer's write pointers */
3442         ring->wptr = 0;
3443         WREG32_SOC15(GC, 0, regCP_RB0_WPTR, lower_32_bits(ring->wptr));
3444         WREG32_SOC15(GC, 0, regCP_RB0_WPTR_HI, upper_32_bits(ring->wptr));
3445
3446         /* set the wb address wether it's enabled or not */
3447         rptr_addr = ring->rptr_gpu_addr;
3448         WREG32_SOC15(GC, 0, regCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr));
3449         WREG32_SOC15(GC, 0, regCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) &
3450                      CP_RB_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK);
3451
3452         wptr_gpu_addr = ring->wptr_gpu_addr;
3453         WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_ADDR_LO,
3454                      lower_32_bits(wptr_gpu_addr));
3455         WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_ADDR_HI,
3456                      upper_32_bits(wptr_gpu_addr));
3457
3458         mdelay(1);
3459         WREG32_SOC15(GC, 0, regCP_RB0_CNTL, tmp);
3460
3461         rb_addr = ring->gpu_addr >> 8;
3462         WREG32_SOC15(GC, 0, regCP_RB0_BASE, rb_addr);
3463         WREG32_SOC15(GC, 0, regCP_RB0_BASE_HI, upper_32_bits(rb_addr));
3464
3465         WREG32_SOC15(GC, 0, regCP_RB_ACTIVE, 1);
3466
3467         gfx_v11_0_cp_gfx_set_doorbell(adev, ring);
3468         mutex_unlock(&adev->srbm_mutex);
3469
3470         /* Init gfx ring 1 for pipe 1 */
3471         if (adev->gfx.num_gfx_rings > 1) {
3472                 mutex_lock(&adev->srbm_mutex);
3473                 gfx_v11_0_cp_gfx_switch_pipe(adev, PIPE_ID1);
3474                 /* maximum supported gfx ring is 2 */
3475                 ring = &adev->gfx.gfx_ring[1];
3476                 rb_bufsz = order_base_2(ring->ring_size / 8);
3477                 tmp = REG_SET_FIELD(0, CP_RB1_CNTL, RB_BUFSZ, rb_bufsz);
3478                 tmp = REG_SET_FIELD(tmp, CP_RB1_CNTL, RB_BLKSZ, rb_bufsz - 2);
3479                 WREG32_SOC15(GC, 0, regCP_RB1_CNTL, tmp);
3480                 /* Initialize the ring buffer's write pointers */
3481                 ring->wptr = 0;
3482                 WREG32_SOC15(GC, 0, regCP_RB1_WPTR, lower_32_bits(ring->wptr));
3483                 WREG32_SOC15(GC, 0, regCP_RB1_WPTR_HI, upper_32_bits(ring->wptr));
3484                 /* Set the wb address wether it's enabled or not */
3485                 rptr_addr = ring->rptr_gpu_addr;
3486                 WREG32_SOC15(GC, 0, regCP_RB1_RPTR_ADDR, lower_32_bits(rptr_addr));
3487                 WREG32_SOC15(GC, 0, regCP_RB1_RPTR_ADDR_HI, upper_32_bits(rptr_addr) &
3488                              CP_RB1_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK);
3489                 wptr_gpu_addr = ring->wptr_gpu_addr;
3490                 WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_ADDR_LO,
3491                              lower_32_bits(wptr_gpu_addr));
3492                 WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_ADDR_HI,
3493                              upper_32_bits(wptr_gpu_addr));
3494
3495                 mdelay(1);
3496                 WREG32_SOC15(GC, 0, regCP_RB1_CNTL, tmp);
3497
3498                 rb_addr = ring->gpu_addr >> 8;
3499                 WREG32_SOC15(GC, 0, regCP_RB1_BASE, rb_addr);
3500                 WREG32_SOC15(GC, 0, regCP_RB1_BASE_HI, upper_32_bits(rb_addr));
3501                 WREG32_SOC15(GC, 0, regCP_RB1_ACTIVE, 1);
3502
3503                 gfx_v11_0_cp_gfx_set_doorbell(adev, ring);
3504                 mutex_unlock(&adev->srbm_mutex);
3505         }
3506         /* Switch to pipe 0 */
3507         mutex_lock(&adev->srbm_mutex);
3508         gfx_v11_0_cp_gfx_switch_pipe(adev, PIPE_ID0);
3509         mutex_unlock(&adev->srbm_mutex);
3510
3511         /* start the ring */
3512         gfx_v11_0_cp_gfx_start(adev);
3513
3514         for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
3515                 ring = &adev->gfx.gfx_ring[i];
3516                 ring->sched.ready = true;
3517         }
3518
3519         return 0;
3520 }
3521
3522 static void gfx_v11_0_cp_compute_enable(struct amdgpu_device *adev, bool enable)
3523 {
3524         u32 data;
3525
3526         if (adev->gfx.rs64_enable) {
3527                 data = RREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL);
3528                 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_INVALIDATE_ICACHE,
3529                                                          enable ? 0 : 1);
3530                 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE0_RESET,
3531                                                          enable ? 0 : 1);
3532                 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE1_RESET,
3533                                                          enable ? 0 : 1);
3534                 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE2_RESET,
3535                                                          enable ? 0 : 1);
3536                 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE3_RESET,
3537                                                          enable ? 0 : 1);
3538                 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE0_ACTIVE,
3539                                                          enable ? 1 : 0);
3540                 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE1_ACTIVE,
3541                                                          enable ? 1 : 0);
3542                 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE2_ACTIVE,
3543                                                          enable ? 1 : 0);
3544                 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE3_ACTIVE,
3545                                                          enable ? 1 : 0);
3546                 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_HALT,
3547                                                          enable ? 0 : 1);
3548                 WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, data);
3549         } else {
3550                 data = RREG32_SOC15(GC, 0, regCP_MEC_CNTL);
3551
3552                 if (enable) {
3553                         data = REG_SET_FIELD(data, CP_MEC_CNTL, MEC_ME1_HALT, 0);
3554                         if (!adev->enable_mes_kiq)
3555                                 data = REG_SET_FIELD(data, CP_MEC_CNTL,
3556                                                      MEC_ME2_HALT, 0);
3557                 } else {
3558                         data = REG_SET_FIELD(data, CP_MEC_CNTL, MEC_ME1_HALT, 1);
3559                         data = REG_SET_FIELD(data, CP_MEC_CNTL, MEC_ME2_HALT, 1);
3560                 }
3561                 WREG32_SOC15(GC, 0, regCP_MEC_CNTL, data);
3562         }
3563
3564         adev->gfx.kiq.ring.sched.ready = enable;
3565
3566         udelay(50);
3567 }
3568
3569 static int gfx_v11_0_cp_compute_load_microcode(struct amdgpu_device *adev)
3570 {
3571         const struct gfx_firmware_header_v1_0 *mec_hdr;
3572         const __le32 *fw_data;
3573         unsigned i, fw_size;
3574         u32 *fw = NULL;
3575         int r;
3576
3577         if (!adev->gfx.mec_fw)
3578                 return -EINVAL;
3579
3580         gfx_v11_0_cp_compute_enable(adev, false);
3581
3582         mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
3583         amdgpu_ucode_print_gfx_hdr(&mec_hdr->header);
3584
3585         fw_data = (const __le32 *)
3586                 (adev->gfx.mec_fw->data +
3587                  le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
3588         fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes);
3589
3590         r = amdgpu_bo_create_reserved(adev, mec_hdr->header.ucode_size_bytes,
3591                                           PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
3592                                           &adev->gfx.mec.mec_fw_obj,
3593                                           &adev->gfx.mec.mec_fw_gpu_addr,
3594                                           (void **)&fw);
3595         if (r) {
3596                 dev_err(adev->dev, "(%d) failed to create mec fw bo\n", r);
3597                 gfx_v11_0_mec_fini(adev);
3598                 return r;
3599         }
3600
3601         memcpy(fw, fw_data, fw_size);
3602
3603         amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj);
3604         amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj);
3605
3606         gfx_v11_0_config_mec_cache(adev, adev->gfx.mec.mec_fw_gpu_addr);
3607
3608         /* MEC1 */
3609         WREG32_SOC15(GC, 0, regCP_MEC_ME1_UCODE_ADDR, 0);
3610
3611         for (i = 0; i < mec_hdr->jt_size; i++)
3612                 WREG32_SOC15(GC, 0, regCP_MEC_ME1_UCODE_DATA,
3613                              le32_to_cpup(fw_data + mec_hdr->jt_offset + i));
3614
3615         WREG32_SOC15(GC, 0, regCP_MEC_ME1_UCODE_ADDR, adev->gfx.mec_fw_version);
3616
3617         return 0;
3618 }
3619
3620 static int gfx_v11_0_cp_compute_load_microcode_rs64(struct amdgpu_device *adev)
3621 {
3622         const struct gfx_firmware_header_v2_0 *mec_hdr;
3623         const __le32 *fw_ucode, *fw_data;
3624         u32 tmp, fw_ucode_size, fw_data_size;
3625         u32 i, usec_timeout = 50000; /* Wait for 50 ms */
3626         u32 *fw_ucode_ptr, *fw_data_ptr;
3627         int r;
3628
3629         if (!adev->gfx.mec_fw)
3630                 return -EINVAL;
3631
3632         gfx_v11_0_cp_compute_enable(adev, false);
3633
3634         mec_hdr = (const struct gfx_firmware_header_v2_0 *)adev->gfx.mec_fw->data;
3635         amdgpu_ucode_print_gfx_hdr(&mec_hdr->header);
3636
3637         fw_ucode = (const __le32 *) (adev->gfx.mec_fw->data +
3638                                 le32_to_cpu(mec_hdr->ucode_offset_bytes));
3639         fw_ucode_size = le32_to_cpu(mec_hdr->ucode_size_bytes);
3640
3641         fw_data = (const __le32 *) (adev->gfx.mec_fw->data +
3642                                 le32_to_cpu(mec_hdr->data_offset_bytes));
3643         fw_data_size = le32_to_cpu(mec_hdr->data_size_bytes);
3644
3645         r = amdgpu_bo_create_reserved(adev, fw_ucode_size,
3646                                       64 * 1024, AMDGPU_GEM_DOMAIN_VRAM,
3647                                       &adev->gfx.mec.mec_fw_obj,
3648                                       &adev->gfx.mec.mec_fw_gpu_addr,
3649                                       (void **)&fw_ucode_ptr);
3650         if (r) {
3651                 dev_err(adev->dev, "(%d) failed to create mec fw ucode bo\n", r);
3652                 gfx_v11_0_mec_fini(adev);
3653                 return r;
3654         }
3655
3656         r = amdgpu_bo_create_reserved(adev, fw_data_size,
3657                                       64 * 1024, AMDGPU_GEM_DOMAIN_VRAM,
3658                                       &adev->gfx.mec.mec_fw_data_obj,
3659                                       &adev->gfx.mec.mec_fw_data_gpu_addr,
3660                                       (void **)&fw_data_ptr);
3661         if (r) {
3662                 dev_err(adev->dev, "(%d) failed to create mec fw ucode bo\n", r);
3663                 gfx_v11_0_mec_fini(adev);
3664                 return r;
3665         }
3666
3667         memcpy(fw_ucode_ptr, fw_ucode, fw_ucode_size);
3668         memcpy(fw_data_ptr, fw_data, fw_data_size);
3669
3670         amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj);
3671         amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_data_obj);
3672         amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj);
3673         amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_data_obj);
3674
3675         tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL);
3676         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0);
3677         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, EXE_DISABLE, 0);
3678         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0);
3679         WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL, tmp);
3680
3681         tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_BASE_CNTL);
3682         tmp = REG_SET_FIELD(tmp, CP_MEC_DC_BASE_CNTL, VMID, 0);
3683         tmp = REG_SET_FIELD(tmp, CP_MEC_DC_BASE_CNTL, CACHE_POLICY, 0);
3684         WREG32_SOC15(GC, 0, regCP_MEC_DC_BASE_CNTL, tmp);
3685
3686         mutex_lock(&adev->srbm_mutex);
3687         for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) {
3688                 soc21_grbm_select(adev, 1, i, 0, 0);
3689
3690                 WREG32_SOC15(GC, 0, regCP_MEC_MDBASE_LO, adev->gfx.mec.mec_fw_data_gpu_addr);
3691                 WREG32_SOC15(GC, 0, regCP_MEC_MDBASE_HI,
3692                      upper_32_bits(adev->gfx.mec.mec_fw_data_gpu_addr));
3693
3694                 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START,
3695                                         mec_hdr->ucode_start_addr_lo >> 2 |
3696                                         mec_hdr->ucode_start_addr_hi << 30);
3697                 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START_HI,
3698                                         mec_hdr->ucode_start_addr_hi >> 2);
3699
3700                 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_LO, adev->gfx.mec.mec_fw_gpu_addr);
3701                 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_HI,
3702                      upper_32_bits(adev->gfx.mec.mec_fw_gpu_addr));
3703         }
3704         mutex_unlock(&adev->srbm_mutex);
3705         soc21_grbm_select(adev, 0, 0, 0, 0);
3706
3707         /* Trigger an invalidation of the L1 instruction caches */
3708         tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL);
3709         tmp = REG_SET_FIELD(tmp, CP_MEC_DC_OP_CNTL, INVALIDATE_DCACHE, 1);
3710         WREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL, tmp);
3711
3712         /* Wait for invalidation complete */
3713         for (i = 0; i < usec_timeout; i++) {
3714                 tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL);
3715                 if (1 == REG_GET_FIELD(tmp, CP_MEC_DC_OP_CNTL,
3716                                        INVALIDATE_DCACHE_COMPLETE))
3717                         break;
3718                 udelay(1);
3719         }
3720
3721         if (i >= usec_timeout) {
3722                 dev_err(adev->dev, "failed to invalidate instruction cache\n");
3723                 return -EINVAL;
3724         }
3725
3726         /* Trigger an invalidation of the L1 instruction caches */
3727         tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL);
3728         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_OP_CNTL, INVALIDATE_CACHE, 1);
3729         WREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL, tmp);
3730
3731         /* Wait for invalidation complete */
3732         for (i = 0; i < usec_timeout; i++) {
3733                 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL);
3734                 if (1 == REG_GET_FIELD(tmp, CP_CPC_IC_OP_CNTL,
3735                                        INVALIDATE_CACHE_COMPLETE))
3736                         break;
3737                 udelay(1);
3738         }
3739
3740         if (i >= usec_timeout) {
3741                 dev_err(adev->dev, "failed to invalidate instruction cache\n");
3742                 return -EINVAL;
3743         }
3744
3745         return 0;
3746 }
3747
3748 static void gfx_v11_0_kiq_setting(struct amdgpu_ring *ring)
3749 {
3750         uint32_t tmp;
3751         struct amdgpu_device *adev = ring->adev;
3752
3753         /* tell RLC which is KIQ queue */
3754         tmp = RREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS);
3755         tmp &= 0xffffff00;
3756         tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue);
3757         WREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS, tmp);
3758         tmp |= 0x80;
3759         WREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS, tmp);
3760 }
3761
3762 static void gfx_v11_0_cp_set_doorbell_range(struct amdgpu_device *adev)
3763 {
3764         /* set graphics engine doorbell range */
3765         WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_RANGE_LOWER,
3766                      (adev->doorbell_index.gfx_ring0 * 2) << 2);
3767         WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_RANGE_UPPER,
3768                      (adev->doorbell_index.gfx_userqueue_end * 2) << 2);
3769
3770         /* set compute engine doorbell range */
3771         WREG32_SOC15(GC, 0, regCP_MEC_DOORBELL_RANGE_LOWER,
3772                      (adev->doorbell_index.kiq * 2) << 2);
3773         WREG32_SOC15(GC, 0, regCP_MEC_DOORBELL_RANGE_UPPER,
3774                      (adev->doorbell_index.userqueue_end * 2) << 2);
3775 }
3776
3777 static int gfx_v11_0_gfx_mqd_init(struct amdgpu_device *adev, void *m,
3778                                   struct amdgpu_mqd_prop *prop)
3779 {
3780         struct v11_gfx_mqd *mqd = m;
3781         uint64_t hqd_gpu_addr, wb_gpu_addr;
3782         uint32_t tmp;
3783         uint32_t rb_bufsz;
3784
3785         /* set up gfx hqd wptr */
3786         mqd->cp_gfx_hqd_wptr = 0;
3787         mqd->cp_gfx_hqd_wptr_hi = 0;
3788
3789         /* set the pointer to the MQD */
3790         mqd->cp_mqd_base_addr = prop->mqd_gpu_addr & 0xfffffffc;
3791         mqd->cp_mqd_base_addr_hi = upper_32_bits(prop->mqd_gpu_addr);
3792
3793         /* set up mqd control */
3794         tmp = RREG32_SOC15(GC, 0, regCP_GFX_MQD_CONTROL);
3795         tmp = REG_SET_FIELD(tmp, CP_GFX_MQD_CONTROL, VMID, 0);
3796         tmp = REG_SET_FIELD(tmp, CP_GFX_MQD_CONTROL, PRIV_STATE, 1);
3797         tmp = REG_SET_FIELD(tmp, CP_GFX_MQD_CONTROL, CACHE_POLICY, 0);
3798         mqd->cp_gfx_mqd_control = tmp;
3799
3800         /* set up gfx_hqd_vimd with 0x0 to indicate the ring buffer's vmid */
3801         tmp = RREG32_SOC15(GC, 0, regCP_GFX_HQD_VMID);
3802         tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_VMID, VMID, 0);
3803         mqd->cp_gfx_hqd_vmid = 0;
3804
3805         /* set up default queue priority level
3806          * 0x0 = low priority, 0x1 = high priority */
3807         tmp = RREG32_SOC15(GC, 0, regCP_GFX_HQD_QUEUE_PRIORITY);
3808         tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_QUEUE_PRIORITY, PRIORITY_LEVEL, 0);
3809         mqd->cp_gfx_hqd_queue_priority = tmp;
3810
3811         /* set up time quantum */
3812         tmp = RREG32_SOC15(GC, 0, regCP_GFX_HQD_QUANTUM);
3813         tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_QUANTUM, QUANTUM_EN, 1);
3814         mqd->cp_gfx_hqd_quantum = tmp;
3815
3816         /* set up gfx hqd base. this is similar as CP_RB_BASE */
3817         hqd_gpu_addr = prop->hqd_base_gpu_addr >> 8;
3818         mqd->cp_gfx_hqd_base = hqd_gpu_addr;
3819         mqd->cp_gfx_hqd_base_hi = upper_32_bits(hqd_gpu_addr);
3820
3821         /* set up hqd_rptr_addr/_hi, similar as CP_RB_RPTR */
3822         wb_gpu_addr = prop->rptr_gpu_addr;
3823         mqd->cp_gfx_hqd_rptr_addr = wb_gpu_addr & 0xfffffffc;
3824         mqd->cp_gfx_hqd_rptr_addr_hi =
3825                 upper_32_bits(wb_gpu_addr) & 0xffff;
3826
3827         /* set up rb_wptr_poll addr */
3828         wb_gpu_addr = prop->wptr_gpu_addr;
3829         mqd->cp_rb_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc;
3830         mqd->cp_rb_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
3831
3832         /* set up the gfx_hqd_control, similar as CP_RB0_CNTL */
3833         rb_bufsz = order_base_2(prop->queue_size / 4) - 1;
3834         tmp = RREG32_SOC15(GC, 0, regCP_GFX_HQD_CNTL);
3835         tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, RB_BUFSZ, rb_bufsz);
3836         tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, RB_BLKSZ, rb_bufsz - 2);
3837 #ifdef __BIG_ENDIAN
3838         tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, BUF_SWAP, 1);
3839 #endif
3840         mqd->cp_gfx_hqd_cntl = tmp;
3841
3842         /* set up cp_doorbell_control */
3843         tmp = RREG32_SOC15(GC, 0, regCP_RB_DOORBELL_CONTROL);
3844         if (prop->use_doorbell) {
3845                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3846                                     DOORBELL_OFFSET, prop->doorbell_index);
3847                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3848                                     DOORBELL_EN, 1);
3849         } else
3850                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3851                                     DOORBELL_EN, 0);
3852         mqd->cp_rb_doorbell_control = tmp;
3853
3854         /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
3855         mqd->cp_gfx_hqd_rptr = RREG32_SOC15(GC, 0, regCP_GFX_HQD_RPTR);
3856
3857         /* active the queue */
3858         mqd->cp_gfx_hqd_active = 1;
3859
3860         return 0;
3861 }
3862
3863 #ifdef BRING_UP_DEBUG
3864 static int gfx_v11_0_gfx_queue_init_register(struct amdgpu_ring *ring)
3865 {
3866         struct amdgpu_device *adev = ring->adev;
3867         struct v11_gfx_mqd *mqd = ring->mqd_ptr;
3868
3869         /* set mmCP_GFX_HQD_WPTR/_HI to 0 */
3870         WREG32_SOC15(GC, 0, regCP_GFX_HQD_WPTR, mqd->cp_gfx_hqd_wptr);
3871         WREG32_SOC15(GC, 0, regCP_GFX_HQD_WPTR_HI, mqd->cp_gfx_hqd_wptr_hi);
3872
3873         /* set GFX_MQD_BASE */
3874         WREG32_SOC15(GC, 0, regCP_MQD_BASE_ADDR, mqd->cp_mqd_base_addr);
3875         WREG32_SOC15(GC, 0, regCP_MQD_BASE_ADDR_HI, mqd->cp_mqd_base_addr_hi);
3876
3877         /* set GFX_MQD_CONTROL */
3878         WREG32_SOC15(GC, 0, regCP_GFX_MQD_CONTROL, mqd->cp_gfx_mqd_control);
3879
3880         /* set GFX_HQD_VMID to 0 */
3881         WREG32_SOC15(GC, 0, regCP_GFX_HQD_VMID, mqd->cp_gfx_hqd_vmid);
3882
3883         WREG32_SOC15(GC, 0, regCP_GFX_HQD_QUEUE_PRIORITY,
3884                         mqd->cp_gfx_hqd_queue_priority);
3885         WREG32_SOC15(GC, 0, regCP_GFX_HQD_QUANTUM, mqd->cp_gfx_hqd_quantum);
3886
3887         /* set GFX_HQD_BASE, similar as CP_RB_BASE */
3888         WREG32_SOC15(GC, 0, regCP_GFX_HQD_BASE, mqd->cp_gfx_hqd_base);
3889         WREG32_SOC15(GC, 0, regCP_GFX_HQD_BASE_HI, mqd->cp_gfx_hqd_base_hi);
3890
3891         /* set GFX_HQD_RPTR_ADDR, similar as CP_RB_RPTR */
3892         WREG32_SOC15(GC, 0, regCP_GFX_HQD_RPTR_ADDR, mqd->cp_gfx_hqd_rptr_addr);
3893         WREG32_SOC15(GC, 0, regCP_GFX_HQD_RPTR_ADDR_HI, mqd->cp_gfx_hqd_rptr_addr_hi);
3894
3895         /* set GFX_HQD_CNTL, similar as CP_RB_CNTL */
3896         WREG32_SOC15(GC, 0, regCP_GFX_HQD_CNTL, mqd->cp_gfx_hqd_cntl);
3897
3898         /* set RB_WPTR_POLL_ADDR */
3899         WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_ADDR_LO, mqd->cp_rb_wptr_poll_addr_lo);
3900         WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_ADDR_HI, mqd->cp_rb_wptr_poll_addr_hi);
3901
3902         /* set RB_DOORBELL_CONTROL */
3903         WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_CONTROL, mqd->cp_rb_doorbell_control);
3904
3905         /* active the queue */
3906         WREG32_SOC15(GC, 0, regCP_GFX_HQD_ACTIVE, mqd->cp_gfx_hqd_active);
3907
3908         return 0;
3909 }
3910 #endif
3911
3912 static int gfx_v11_0_gfx_init_queue(struct amdgpu_ring *ring)
3913 {
3914         struct amdgpu_device *adev = ring->adev;
3915         struct v11_gfx_mqd *mqd = ring->mqd_ptr;
3916         int mqd_idx = ring - &adev->gfx.gfx_ring[0];
3917
3918         if (!amdgpu_in_reset(adev) && !adev->in_suspend) {
3919                 memset((void *)mqd, 0, sizeof(*mqd));
3920                 mutex_lock(&adev->srbm_mutex);
3921                 soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
3922                 amdgpu_ring_init_mqd(ring);
3923 #ifdef BRING_UP_DEBUG
3924                 gfx_v11_0_gfx_queue_init_register(ring);
3925 #endif
3926                 soc21_grbm_select(adev, 0, 0, 0, 0);
3927                 mutex_unlock(&adev->srbm_mutex);
3928                 if (adev->gfx.me.mqd_backup[mqd_idx])
3929                         memcpy(adev->gfx.me.mqd_backup[mqd_idx], mqd, sizeof(*mqd));
3930         } else if (amdgpu_in_reset(adev)) {
3931                 /* reset mqd with the backup copy */
3932                 if (adev->gfx.me.mqd_backup[mqd_idx])
3933                         memcpy(mqd, adev->gfx.me.mqd_backup[mqd_idx], sizeof(*mqd));
3934                 /* reset the ring */
3935                 ring->wptr = 0;
3936                 *ring->wptr_cpu_addr = 0;
3937                 amdgpu_ring_clear_ring(ring);
3938 #ifdef BRING_UP_DEBUG
3939                 mutex_lock(&adev->srbm_mutex);
3940                 soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
3941                 gfx_v11_0_gfx_queue_init_register(ring);
3942                 soc21_grbm_select(adev, 0, 0, 0, 0);
3943                 mutex_unlock(&adev->srbm_mutex);
3944 #endif
3945         } else {
3946                 amdgpu_ring_clear_ring(ring);
3947         }
3948
3949         return 0;
3950 }
3951
3952 #ifndef BRING_UP_DEBUG
3953 static int gfx_v11_0_kiq_enable_kgq(struct amdgpu_device *adev)
3954 {
3955         struct amdgpu_kiq *kiq = &adev->gfx.kiq;
3956         struct amdgpu_ring *kiq_ring = &adev->gfx.kiq.ring;
3957         int r, i;
3958
3959         if (!kiq->pmf || !kiq->pmf->kiq_map_queues)
3960                 return -EINVAL;
3961
3962         r = amdgpu_ring_alloc(kiq_ring, kiq->pmf->map_queues_size *
3963                                         adev->gfx.num_gfx_rings);
3964         if (r) {
3965                 DRM_ERROR("Failed to lock KIQ (%d).\n", r);
3966                 return r;
3967         }
3968
3969         for (i = 0; i < adev->gfx.num_gfx_rings; i++)
3970                 kiq->pmf->kiq_map_queues(kiq_ring, &adev->gfx.gfx_ring[i]);
3971
3972         return amdgpu_ring_test_helper(kiq_ring);
3973 }
3974 #endif
3975
3976 static int gfx_v11_0_cp_async_gfx_ring_resume(struct amdgpu_device *adev)
3977 {
3978         int r, i;
3979         struct amdgpu_ring *ring;
3980
3981         for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
3982                 ring = &adev->gfx.gfx_ring[i];
3983
3984                 r = amdgpu_bo_reserve(ring->mqd_obj, false);
3985                 if (unlikely(r != 0))
3986                         goto done;
3987
3988                 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
3989                 if (!r) {
3990                         r = gfx_v11_0_gfx_init_queue(ring);
3991                         amdgpu_bo_kunmap(ring->mqd_obj);
3992                         ring->mqd_ptr = NULL;
3993                 }
3994                 amdgpu_bo_unreserve(ring->mqd_obj);
3995                 if (r)
3996                         goto done;
3997         }
3998 #ifndef BRING_UP_DEBUG
3999         r = gfx_v11_0_kiq_enable_kgq(adev);
4000         if (r)
4001                 goto done;
4002 #endif
4003         r = gfx_v11_0_cp_gfx_start(adev);
4004         if (r)
4005                 goto done;
4006
4007         for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
4008                 ring = &adev->gfx.gfx_ring[i];
4009                 ring->sched.ready = true;
4010         }
4011 done:
4012         return r;
4013 }
4014
4015 static int gfx_v11_0_compute_mqd_init(struct amdgpu_device *adev, void *m,
4016                                       struct amdgpu_mqd_prop *prop)
4017 {
4018         struct v11_compute_mqd *mqd = m;
4019         uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr;
4020         uint32_t tmp;
4021
4022         mqd->header = 0xC0310800;
4023         mqd->compute_pipelinestat_enable = 0x00000001;
4024         mqd->compute_static_thread_mgmt_se0 = 0xffffffff;
4025         mqd->compute_static_thread_mgmt_se1 = 0xffffffff;
4026         mqd->compute_static_thread_mgmt_se2 = 0xffffffff;
4027         mqd->compute_static_thread_mgmt_se3 = 0xffffffff;
4028         mqd->compute_misc_reserved = 0x00000007;
4029
4030         eop_base_addr = prop->eop_gpu_addr >> 8;
4031         mqd->cp_hqd_eop_base_addr_lo = eop_base_addr;
4032         mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr);
4033
4034         /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4035         tmp = RREG32_SOC15(GC, 0, regCP_HQD_EOP_CONTROL);
4036         tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE,
4037                         (order_base_2(GFX11_MEC_HPD_SIZE / 4) - 1));
4038
4039         mqd->cp_hqd_eop_control = tmp;
4040
4041         /* enable doorbell? */
4042         tmp = RREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL);
4043
4044         if (prop->use_doorbell) {
4045                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4046                                     DOORBELL_OFFSET, prop->doorbell_index);
4047                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4048                                     DOORBELL_EN, 1);
4049                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4050                                     DOORBELL_SOURCE, 0);
4051                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4052                                     DOORBELL_HIT, 0);
4053         } else {
4054                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4055                                     DOORBELL_EN, 0);
4056         }
4057
4058         mqd->cp_hqd_pq_doorbell_control = tmp;
4059
4060         /* disable the queue if it's active */
4061         mqd->cp_hqd_dequeue_request = 0;
4062         mqd->cp_hqd_pq_rptr = 0;
4063         mqd->cp_hqd_pq_wptr_lo = 0;
4064         mqd->cp_hqd_pq_wptr_hi = 0;
4065
4066         /* set the pointer to the MQD */
4067         mqd->cp_mqd_base_addr_lo = prop->mqd_gpu_addr & 0xfffffffc;
4068         mqd->cp_mqd_base_addr_hi = upper_32_bits(prop->mqd_gpu_addr);
4069
4070         /* set MQD vmid to 0 */
4071         tmp = RREG32_SOC15(GC, 0, regCP_MQD_CONTROL);
4072         tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0);
4073         mqd->cp_mqd_control = tmp;
4074
4075         /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
4076         hqd_gpu_addr = prop->hqd_base_gpu_addr >> 8;
4077         mqd->cp_hqd_pq_base_lo = hqd_gpu_addr;
4078         mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
4079
4080         /* set up the HQD, this is similar to CP_RB0_CNTL */
4081         tmp = RREG32_SOC15(GC, 0, regCP_HQD_PQ_CONTROL);
4082         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE,
4083                             (order_base_2(prop->queue_size / 4) - 1));
4084         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE,
4085                             (order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1));
4086         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 0);
4087         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, TUNNEL_DISPATCH, 0);
4088         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1);
4089         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1);
4090         mqd->cp_hqd_pq_control = tmp;
4091
4092         /* set the wb address whether it's enabled or not */
4093         wb_gpu_addr = prop->rptr_gpu_addr;
4094         mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc;
4095         mqd->cp_hqd_pq_rptr_report_addr_hi =
4096                 upper_32_bits(wb_gpu_addr) & 0xffff;
4097
4098         /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
4099         wb_gpu_addr = prop->wptr_gpu_addr;
4100         mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc;
4101         mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
4102
4103         tmp = 0;
4104         /* enable the doorbell if requested */
4105         if (prop->use_doorbell) {
4106                 tmp = RREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL);
4107                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4108                                 DOORBELL_OFFSET, prop->doorbell_index);
4109
4110                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4111                                     DOORBELL_EN, 1);
4112                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4113                                     DOORBELL_SOURCE, 0);
4114                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4115                                     DOORBELL_HIT, 0);
4116         }
4117
4118         mqd->cp_hqd_pq_doorbell_control = tmp;
4119
4120         /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
4121         mqd->cp_hqd_pq_rptr = RREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR);
4122
4123         /* set the vmid for the queue */
4124         mqd->cp_hqd_vmid = 0;
4125
4126         tmp = RREG32_SOC15(GC, 0, regCP_HQD_PERSISTENT_STATE);
4127         tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x55);
4128         mqd->cp_hqd_persistent_state = tmp;
4129
4130         /* set MIN_IB_AVAIL_SIZE */
4131         tmp = RREG32_SOC15(GC, 0, regCP_HQD_IB_CONTROL);
4132         tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 3);
4133         mqd->cp_hqd_ib_control = tmp;
4134
4135         /* set static priority for a compute queue/ring */
4136         mqd->cp_hqd_pipe_priority = prop->hqd_pipe_priority;
4137         mqd->cp_hqd_queue_priority = prop->hqd_queue_priority;
4138
4139         mqd->cp_hqd_active = prop->hqd_active;
4140
4141         return 0;
4142 }
4143
4144 static int gfx_v11_0_kiq_init_register(struct amdgpu_ring *ring)
4145 {
4146         struct amdgpu_device *adev = ring->adev;
4147         struct v11_compute_mqd *mqd = ring->mqd_ptr;
4148         int j;
4149
4150         /* inactivate the queue */
4151         if (amdgpu_sriov_vf(adev))
4152                 WREG32_SOC15(GC, 0, regCP_HQD_ACTIVE, 0);
4153
4154         /* disable wptr polling */
4155         WREG32_FIELD15_PREREG(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0);
4156
4157         /* write the EOP addr */
4158         WREG32_SOC15(GC, 0, regCP_HQD_EOP_BASE_ADDR,
4159                mqd->cp_hqd_eop_base_addr_lo);
4160         WREG32_SOC15(GC, 0, regCP_HQD_EOP_BASE_ADDR_HI,
4161                mqd->cp_hqd_eop_base_addr_hi);
4162
4163         /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4164         WREG32_SOC15(GC, 0, regCP_HQD_EOP_CONTROL,
4165                mqd->cp_hqd_eop_control);
4166
4167         /* enable doorbell? */
4168         WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL,
4169                mqd->cp_hqd_pq_doorbell_control);
4170
4171         /* disable the queue if it's active */
4172         if (RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) & 1) {
4173                 WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST, 1);
4174                 for (j = 0; j < adev->usec_timeout; j++) {
4175                         if (!(RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) & 1))
4176                                 break;
4177                         udelay(1);
4178                 }
4179                 WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST,
4180                        mqd->cp_hqd_dequeue_request);
4181                 WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR,
4182                        mqd->cp_hqd_pq_rptr);
4183                 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_LO,
4184                        mqd->cp_hqd_pq_wptr_lo);
4185                 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_HI,
4186                        mqd->cp_hqd_pq_wptr_hi);
4187         }
4188
4189         /* set the pointer to the MQD */
4190         WREG32_SOC15(GC, 0, regCP_MQD_BASE_ADDR,
4191                mqd->cp_mqd_base_addr_lo);
4192         WREG32_SOC15(GC, 0, regCP_MQD_BASE_ADDR_HI,
4193                mqd->cp_mqd_base_addr_hi);
4194
4195         /* set MQD vmid to 0 */
4196         WREG32_SOC15(GC, 0, regCP_MQD_CONTROL,
4197                mqd->cp_mqd_control);
4198
4199         /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
4200         WREG32_SOC15(GC, 0, regCP_HQD_PQ_BASE,
4201                mqd->cp_hqd_pq_base_lo);
4202         WREG32_SOC15(GC, 0, regCP_HQD_PQ_BASE_HI,
4203                mqd->cp_hqd_pq_base_hi);
4204
4205         /* set up the HQD, this is similar to CP_RB0_CNTL */
4206         WREG32_SOC15(GC, 0, regCP_HQD_PQ_CONTROL,
4207                mqd->cp_hqd_pq_control);
4208
4209         /* set the wb address whether it's enabled or not */
4210         WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR_REPORT_ADDR,
4211                 mqd->cp_hqd_pq_rptr_report_addr_lo);
4212         WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR_REPORT_ADDR_HI,
4213                 mqd->cp_hqd_pq_rptr_report_addr_hi);
4214
4215         /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
4216         WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR,
4217                mqd->cp_hqd_pq_wptr_poll_addr_lo);
4218         WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR_HI,
4219                mqd->cp_hqd_pq_wptr_poll_addr_hi);
4220
4221         /* enable the doorbell if requested */
4222         if (ring->use_doorbell) {
4223                 WREG32_SOC15(GC, 0, regCP_MEC_DOORBELL_RANGE_LOWER,
4224                         (adev->doorbell_index.kiq * 2) << 2);
4225                 WREG32_SOC15(GC, 0, regCP_MEC_DOORBELL_RANGE_UPPER,
4226                         (adev->doorbell_index.userqueue_end * 2) << 2);
4227         }
4228
4229         WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL,
4230                mqd->cp_hqd_pq_doorbell_control);
4231
4232         /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
4233         WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_LO,
4234                mqd->cp_hqd_pq_wptr_lo);
4235         WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_HI,
4236                mqd->cp_hqd_pq_wptr_hi);
4237
4238         /* set the vmid for the queue */
4239         WREG32_SOC15(GC, 0, regCP_HQD_VMID, mqd->cp_hqd_vmid);
4240
4241         WREG32_SOC15(GC, 0, regCP_HQD_PERSISTENT_STATE,
4242                mqd->cp_hqd_persistent_state);
4243
4244         /* activate the queue */
4245         WREG32_SOC15(GC, 0, regCP_HQD_ACTIVE,
4246                mqd->cp_hqd_active);
4247
4248         if (ring->use_doorbell)
4249                 WREG32_FIELD15_PREREG(GC, 0, CP_PQ_STATUS, DOORBELL_ENABLE, 1);
4250
4251         return 0;
4252 }
4253
4254 static int gfx_v11_0_kiq_init_queue(struct amdgpu_ring *ring)
4255 {
4256         struct amdgpu_device *adev = ring->adev;
4257         struct v11_compute_mqd *mqd = ring->mqd_ptr;
4258         int mqd_idx = AMDGPU_MAX_COMPUTE_RINGS;
4259
4260         gfx_v11_0_kiq_setting(ring);
4261
4262         if (amdgpu_in_reset(adev)) { /* for GPU_RESET case */
4263                 /* reset MQD to a clean status */
4264                 if (adev->gfx.mec.mqd_backup[mqd_idx])
4265                         memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(*mqd));
4266
4267                 /* reset ring buffer */
4268                 ring->wptr = 0;
4269                 amdgpu_ring_clear_ring(ring);
4270
4271                 mutex_lock(&adev->srbm_mutex);
4272                 soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
4273                 gfx_v11_0_kiq_init_register(ring);
4274                 soc21_grbm_select(adev, 0, 0, 0, 0);
4275                 mutex_unlock(&adev->srbm_mutex);
4276         } else {
4277                 memset((void *)mqd, 0, sizeof(*mqd));
4278                 mutex_lock(&adev->srbm_mutex);
4279                 soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
4280                 amdgpu_ring_init_mqd(ring);
4281                 gfx_v11_0_kiq_init_register(ring);
4282                 soc21_grbm_select(adev, 0, 0, 0, 0);
4283                 mutex_unlock(&adev->srbm_mutex);
4284
4285                 if (adev->gfx.mec.mqd_backup[mqd_idx])
4286                         memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(*mqd));
4287         }
4288
4289         return 0;
4290 }
4291
4292 static int gfx_v11_0_kcq_init_queue(struct amdgpu_ring *ring)
4293 {
4294         struct amdgpu_device *adev = ring->adev;
4295         struct v11_compute_mqd *mqd = ring->mqd_ptr;
4296         int mqd_idx = ring - &adev->gfx.compute_ring[0];
4297
4298         if (!amdgpu_in_reset(adev) && !adev->in_suspend) {
4299                 memset((void *)mqd, 0, sizeof(*mqd));
4300                 mutex_lock(&adev->srbm_mutex);
4301                 soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
4302                 amdgpu_ring_init_mqd(ring);
4303                 soc21_grbm_select(adev, 0, 0, 0, 0);
4304                 mutex_unlock(&adev->srbm_mutex);
4305
4306                 if (adev->gfx.mec.mqd_backup[mqd_idx])
4307                         memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(*mqd));
4308         } else if (amdgpu_in_reset(adev)) { /* for GPU_RESET case */
4309                 /* reset MQD to a clean status */
4310                 if (adev->gfx.mec.mqd_backup[mqd_idx])
4311                         memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(*mqd));
4312
4313                 /* reset ring buffer */
4314                 ring->wptr = 0;
4315                 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, 0);
4316                 amdgpu_ring_clear_ring(ring);
4317         } else {
4318                 amdgpu_ring_clear_ring(ring);
4319         }
4320
4321         return 0;
4322 }
4323
4324 static int gfx_v11_0_kiq_resume(struct amdgpu_device *adev)
4325 {
4326         struct amdgpu_ring *ring;
4327         int r;
4328
4329         ring = &adev->gfx.kiq.ring;
4330
4331         r = amdgpu_bo_reserve(ring->mqd_obj, false);
4332         if (unlikely(r != 0))
4333                 return r;
4334
4335         r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
4336         if (unlikely(r != 0)) {
4337                 amdgpu_bo_unreserve(ring->mqd_obj);
4338                 return r;
4339         }
4340
4341         gfx_v11_0_kiq_init_queue(ring);
4342         amdgpu_bo_kunmap(ring->mqd_obj);
4343         ring->mqd_ptr = NULL;
4344         amdgpu_bo_unreserve(ring->mqd_obj);
4345         ring->sched.ready = true;
4346         return 0;
4347 }
4348
4349 static int gfx_v11_0_kcq_resume(struct amdgpu_device *adev)
4350 {
4351         struct amdgpu_ring *ring = NULL;
4352         int r = 0, i;
4353
4354         if (!amdgpu_async_gfx_ring)
4355                 gfx_v11_0_cp_compute_enable(adev, true);
4356
4357         for (i = 0; i < adev->gfx.num_compute_rings; i++) {
4358                 ring = &adev->gfx.compute_ring[i];
4359
4360                 r = amdgpu_bo_reserve(ring->mqd_obj, false);
4361                 if (unlikely(r != 0))
4362                         goto done;
4363                 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
4364                 if (!r) {
4365                         r = gfx_v11_0_kcq_init_queue(ring);
4366                         amdgpu_bo_kunmap(ring->mqd_obj);
4367                         ring->mqd_ptr = NULL;
4368                 }
4369                 amdgpu_bo_unreserve(ring->mqd_obj);
4370                 if (r)
4371                         goto done;
4372         }
4373
4374         r = amdgpu_gfx_enable_kcq(adev);
4375 done:
4376         return r;
4377 }
4378
4379 static int gfx_v11_0_cp_resume(struct amdgpu_device *adev)
4380 {
4381         int r, i;
4382         struct amdgpu_ring *ring;
4383
4384         if (!(adev->flags & AMD_IS_APU))
4385                 gfx_v11_0_enable_gui_idle_interrupt(adev, false);
4386
4387         if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
4388                 /* legacy firmware loading */
4389                 r = gfx_v11_0_cp_gfx_load_microcode(adev);
4390                 if (r)
4391                         return r;
4392
4393                 if (adev->gfx.rs64_enable)
4394                         r = gfx_v11_0_cp_compute_load_microcode_rs64(adev);
4395                 else
4396                         r = gfx_v11_0_cp_compute_load_microcode(adev);
4397                 if (r)
4398                         return r;
4399         }
4400
4401         gfx_v11_0_cp_set_doorbell_range(adev);
4402
4403         if (amdgpu_async_gfx_ring) {
4404                 gfx_v11_0_cp_compute_enable(adev, true);
4405                 gfx_v11_0_cp_gfx_enable(adev, true);
4406         }
4407
4408         if (adev->enable_mes_kiq && adev->mes.kiq_hw_init)
4409                 r = amdgpu_mes_kiq_hw_init(adev);
4410         else
4411                 r = gfx_v11_0_kiq_resume(adev);
4412         if (r)
4413                 return r;
4414
4415         r = gfx_v11_0_kcq_resume(adev);
4416         if (r)
4417                 return r;
4418
4419         if (!amdgpu_async_gfx_ring) {
4420                 r = gfx_v11_0_cp_gfx_resume(adev);
4421                 if (r)
4422                         return r;
4423         } else {
4424                 r = gfx_v11_0_cp_async_gfx_ring_resume(adev);
4425                 if (r)
4426                         return r;
4427         }
4428
4429         for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
4430                 ring = &adev->gfx.gfx_ring[i];
4431                 r = amdgpu_ring_test_helper(ring);
4432                 if (r)
4433                         return r;
4434         }
4435
4436         for (i = 0; i < adev->gfx.num_compute_rings; i++) {
4437                 ring = &adev->gfx.compute_ring[i];
4438                 r = amdgpu_ring_test_helper(ring);
4439                 if (r)
4440                         return r;
4441         }
4442
4443         return 0;
4444 }
4445
4446 static void gfx_v11_0_cp_enable(struct amdgpu_device *adev, bool enable)
4447 {
4448         gfx_v11_0_cp_gfx_enable(adev, enable);
4449         gfx_v11_0_cp_compute_enable(adev, enable);
4450 }
4451
4452 static int gfx_v11_0_gfxhub_enable(struct amdgpu_device *adev)
4453 {
4454         int r;
4455         bool value;
4456
4457         r = adev->gfxhub.funcs->gart_enable(adev);
4458         if (r)
4459                 return r;
4460
4461         adev->hdp.funcs->flush_hdp(adev, NULL);
4462
4463         value = (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS) ?
4464                 false : true;
4465
4466         adev->gfxhub.funcs->set_fault_enable_default(adev, value);
4467         amdgpu_gmc_flush_gpu_tlb(adev, 0, AMDGPU_GFXHUB_0, 0);
4468
4469         return 0;
4470 }
4471
4472 static void gfx_v11_0_select_cp_fw_arch(struct amdgpu_device *adev)
4473 {
4474         u32 tmp;
4475
4476         /* select RS64 */
4477         if (adev->gfx.rs64_enable) {
4478                 tmp = RREG32_SOC15(GC, 0, regCP_GFX_CNTL);
4479                 tmp = REG_SET_FIELD(tmp, CP_GFX_CNTL, ENGINE_SEL, 1);
4480                 WREG32_SOC15(GC, 0, regCP_GFX_CNTL, tmp);
4481
4482                 tmp = RREG32_SOC15(GC, 0, regCP_MEC_ISA_CNTL);
4483                 tmp = REG_SET_FIELD(tmp, CP_MEC_ISA_CNTL, ISA_MODE, 1);
4484                 WREG32_SOC15(GC, 0, regCP_MEC_ISA_CNTL, tmp);
4485         }
4486
4487         if (amdgpu_emu_mode == 1)
4488                 msleep(100);
4489 }
4490
4491 static int get_gb_addr_config(struct amdgpu_device * adev)
4492 {
4493         u32 gb_addr_config;
4494
4495         gb_addr_config = RREG32_SOC15(GC, 0, regGB_ADDR_CONFIG);
4496         if (gb_addr_config == 0)
4497                 return -EINVAL;
4498
4499         adev->gfx.config.gb_addr_config_fields.num_pkrs =
4500                 1 << REG_GET_FIELD(gb_addr_config, GB_ADDR_CONFIG, NUM_PKRS);
4501
4502         adev->gfx.config.gb_addr_config = gb_addr_config;
4503
4504         adev->gfx.config.gb_addr_config_fields.num_pipes = 1 <<
4505                         REG_GET_FIELD(adev->gfx.config.gb_addr_config,
4506                                       GB_ADDR_CONFIG, NUM_PIPES);
4507
4508         adev->gfx.config.max_tile_pipes =
4509                 adev->gfx.config.gb_addr_config_fields.num_pipes;
4510
4511         adev->gfx.config.gb_addr_config_fields.max_compress_frags = 1 <<
4512                         REG_GET_FIELD(adev->gfx.config.gb_addr_config,
4513                                       GB_ADDR_CONFIG, MAX_COMPRESSED_FRAGS);
4514         adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 1 <<
4515                         REG_GET_FIELD(adev->gfx.config.gb_addr_config,
4516                                       GB_ADDR_CONFIG, NUM_RB_PER_SE);
4517         adev->gfx.config.gb_addr_config_fields.num_se = 1 <<
4518                         REG_GET_FIELD(adev->gfx.config.gb_addr_config,
4519                                       GB_ADDR_CONFIG, NUM_SHADER_ENGINES);
4520         adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 1 << (8 +
4521                         REG_GET_FIELD(adev->gfx.config.gb_addr_config,
4522                                       GB_ADDR_CONFIG, PIPE_INTERLEAVE_SIZE));
4523
4524         return 0;
4525 }
4526
4527 static void gfx_v11_0_disable_gpa_mode(struct amdgpu_device *adev)
4528 {
4529         uint32_t data;
4530
4531         data = RREG32_SOC15(GC, 0, regCPC_PSP_DEBUG);
4532         data |= CPC_PSP_DEBUG__GPA_OVERRIDE_MASK;
4533         WREG32_SOC15(GC, 0, regCPC_PSP_DEBUG, data);
4534
4535         data = RREG32_SOC15(GC, 0, regCPG_PSP_DEBUG);
4536         data |= CPG_PSP_DEBUG__GPA_OVERRIDE_MASK;
4537         WREG32_SOC15(GC, 0, regCPG_PSP_DEBUG, data);
4538 }
4539
4540 static int gfx_v11_0_hw_init(void *handle)
4541 {
4542         int r;
4543         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4544
4545         if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) {
4546                 if (adev->gfx.imu.funcs) {
4547                         /* RLC autoload sequence 1: Program rlc ram */
4548                         if (adev->gfx.imu.funcs->program_rlc_ram)
4549                                 adev->gfx.imu.funcs->program_rlc_ram(adev);
4550                 }
4551                 /* rlc autoload firmware */
4552                 r = gfx_v11_0_rlc_backdoor_autoload_enable(adev);
4553                 if (r)
4554                         return r;
4555         } else {
4556                 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
4557                         if (adev->gfx.imu.funcs && (amdgpu_dpm > 0)) {
4558                                 if (adev->gfx.imu.funcs->load_microcode)
4559                                         adev->gfx.imu.funcs->load_microcode(adev);
4560                                 if (adev->gfx.imu.funcs->setup_imu)
4561                                         adev->gfx.imu.funcs->setup_imu(adev);
4562                                 if (adev->gfx.imu.funcs->start_imu)
4563                                         adev->gfx.imu.funcs->start_imu(adev);
4564                         }
4565                 }
4566         }
4567
4568         if ((adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) ||
4569             (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP)) {
4570                 r = gfx_v11_0_wait_for_rlc_autoload_complete(adev);
4571                 if (r) {
4572                         dev_err(adev->dev, "(%d) failed to wait rlc autoload complete\n", r);
4573                         return r;
4574                 }
4575         }
4576
4577         adev->gfx.is_poweron = true;
4578
4579         if(get_gb_addr_config(adev))
4580                 DRM_WARN("Invalid gb_addr_config !\n");
4581
4582         if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP &&
4583             adev->gfx.rs64_enable)
4584                 gfx_v11_0_config_gfx_rs64(adev);
4585
4586         r = gfx_v11_0_gfxhub_enable(adev);
4587         if (r)
4588                 return r;
4589
4590         if (!amdgpu_emu_mode)
4591                 gfx_v11_0_init_golden_registers(adev);
4592
4593         if ((adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) ||
4594             (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO && amdgpu_dpm == 1)) {
4595                 /**
4596                  * For gfx 11, rlc firmware loading relies on smu firmware is
4597                  * loaded firstly, so in direct type, it has to load smc ucode
4598                  * here before rlc.
4599                  */
4600                 if (!(adev->flags & AMD_IS_APU)) {
4601                         r = amdgpu_pm_load_smu_firmware(adev, NULL);
4602                         if (r)
4603                                 return r;
4604                 }
4605         }
4606
4607         gfx_v11_0_constants_init(adev);
4608
4609         if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP)
4610                 gfx_v11_0_select_cp_fw_arch(adev);
4611
4612         if (adev->nbio.funcs->gc_doorbell_init)
4613                 adev->nbio.funcs->gc_doorbell_init(adev);
4614
4615         r = gfx_v11_0_rlc_resume(adev);
4616         if (r)
4617                 return r;
4618
4619         /*
4620          * init golden registers and rlc resume may override some registers,
4621          * reconfig them here
4622          */
4623         gfx_v11_0_tcp_harvest(adev);
4624
4625         r = gfx_v11_0_cp_resume(adev);
4626         if (r)
4627                 return r;
4628
4629         return r;
4630 }
4631
4632 #ifndef BRING_UP_DEBUG
4633 static int gfx_v11_0_kiq_disable_kgq(struct amdgpu_device *adev)
4634 {
4635         struct amdgpu_kiq *kiq = &adev->gfx.kiq;
4636         struct amdgpu_ring *kiq_ring = &kiq->ring;
4637         int i, r = 0;
4638
4639         if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues)
4640                 return -EINVAL;
4641
4642         if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size *
4643                                         adev->gfx.num_gfx_rings))
4644                 return -ENOMEM;
4645
4646         for (i = 0; i < adev->gfx.num_gfx_rings; i++)
4647                 kiq->pmf->kiq_unmap_queues(kiq_ring, &adev->gfx.gfx_ring[i],
4648                                            PREEMPT_QUEUES, 0, 0);
4649
4650         if (adev->gfx.kiq.ring.sched.ready)
4651                 r = amdgpu_ring_test_helper(kiq_ring);
4652
4653         return r;
4654 }
4655 #endif
4656
4657 static int gfx_v11_0_hw_fini(void *handle)
4658 {
4659         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4660         int r;
4661         uint32_t tmp;
4662
4663         amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
4664         amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
4665
4666         if (!adev->no_hw_access) {
4667 #ifndef BRING_UP_DEBUG
4668                 if (amdgpu_async_gfx_ring) {
4669                         r = gfx_v11_0_kiq_disable_kgq(adev);
4670                         if (r)
4671                                 DRM_ERROR("KGQ disable failed\n");
4672                 }
4673 #endif
4674                 if (amdgpu_gfx_disable_kcq(adev))
4675                         DRM_ERROR("KCQ disable failed\n");
4676
4677                 amdgpu_mes_kiq_hw_fini(adev);
4678         }
4679
4680         if (amdgpu_sriov_vf(adev)) {
4681                 gfx_v11_0_cp_gfx_enable(adev, false);
4682                 /* Program KIQ position of RLC_CP_SCHEDULERS during destroy */
4683                 tmp = RREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS);
4684                 tmp &= 0xffffff00;
4685                 WREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS, tmp);
4686
4687                 return 0;
4688         }
4689         gfx_v11_0_cp_enable(adev, false);
4690         gfx_v11_0_enable_gui_idle_interrupt(adev, false);
4691
4692         adev->gfxhub.funcs->gart_disable(adev);
4693
4694         adev->gfx.is_poweron = false;
4695
4696         return 0;
4697 }
4698
4699 static int gfx_v11_0_suspend(void *handle)
4700 {
4701         return gfx_v11_0_hw_fini(handle);
4702 }
4703
4704 static int gfx_v11_0_resume(void *handle)
4705 {
4706         return gfx_v11_0_hw_init(handle);
4707 }
4708
4709 static bool gfx_v11_0_is_idle(void *handle)
4710 {
4711         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4712
4713         if (REG_GET_FIELD(RREG32_SOC15(GC, 0, regGRBM_STATUS),
4714                                 GRBM_STATUS, GUI_ACTIVE))
4715                 return false;
4716         else
4717                 return true;
4718 }
4719
4720 static int gfx_v11_0_wait_for_idle(void *handle)
4721 {
4722         unsigned i;
4723         u32 tmp;
4724         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4725
4726         for (i = 0; i < adev->usec_timeout; i++) {
4727                 /* read MC_STATUS */
4728                 tmp = RREG32_SOC15(GC, 0, regGRBM_STATUS) &
4729                         GRBM_STATUS__GUI_ACTIVE_MASK;
4730
4731                 if (!REG_GET_FIELD(tmp, GRBM_STATUS, GUI_ACTIVE))
4732                         return 0;
4733                 udelay(1);
4734         }
4735         return -ETIMEDOUT;
4736 }
4737
4738 static int gfx_v11_0_soft_reset(void *handle)
4739 {
4740         u32 grbm_soft_reset = 0;
4741         u32 tmp;
4742         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4743
4744         /* GRBM_STATUS */
4745         tmp = RREG32_SOC15(GC, 0, regGRBM_STATUS);
4746         if (tmp & (GRBM_STATUS__PA_BUSY_MASK | GRBM_STATUS__SC_BUSY_MASK |
4747                    GRBM_STATUS__BCI_BUSY_MASK | GRBM_STATUS__SX_BUSY_MASK |
4748                    GRBM_STATUS__TA_BUSY_MASK | GRBM_STATUS__DB_BUSY_MASK |
4749                    GRBM_STATUS__CB_BUSY_MASK | GRBM_STATUS__GDS_BUSY_MASK |
4750                    GRBM_STATUS__SPI_BUSY_MASK | GRBM_STATUS__GE_BUSY_NO_DMA_MASK)) {
4751                 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4752                                                 GRBM_SOFT_RESET, SOFT_RESET_CP,
4753                                                 1);
4754                 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4755                                                 GRBM_SOFT_RESET, SOFT_RESET_GFX,
4756                                                 1);
4757         }
4758
4759         if (tmp & (GRBM_STATUS__CP_BUSY_MASK | GRBM_STATUS__CP_COHERENCY_BUSY_MASK)) {
4760                 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4761                                                 GRBM_SOFT_RESET, SOFT_RESET_CP,
4762                                                 1);
4763         }
4764
4765         /* GRBM_STATUS2 */
4766         tmp = RREG32_SOC15(GC, 0, regGRBM_STATUS2);
4767         if (REG_GET_FIELD(tmp, GRBM_STATUS2, RLC_BUSY))
4768                 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4769                                                 GRBM_SOFT_RESET,
4770                                                 SOFT_RESET_RLC,
4771                                                 1);
4772
4773         if (grbm_soft_reset) {
4774                 /* stop the rlc */
4775                 gfx_v11_0_rlc_stop(adev);
4776
4777                 /* Disable GFX parsing/prefetching */
4778                 gfx_v11_0_cp_gfx_enable(adev, false);
4779
4780                 /* Disable MEC parsing/prefetching */
4781                 gfx_v11_0_cp_compute_enable(adev, false);
4782
4783                 if (grbm_soft_reset) {
4784                         tmp = RREG32_SOC15(GC, 0, regGRBM_SOFT_RESET);
4785                         tmp |= grbm_soft_reset;
4786                         dev_info(adev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
4787                         WREG32_SOC15(GC, 0, regGRBM_SOFT_RESET, tmp);
4788                         tmp = RREG32_SOC15(GC, 0, regGRBM_SOFT_RESET);
4789
4790                         udelay(50);
4791
4792                         tmp &= ~grbm_soft_reset;
4793                         WREG32_SOC15(GC, 0, regGRBM_SOFT_RESET, tmp);
4794                         tmp = RREG32_SOC15(GC, 0, regGRBM_SOFT_RESET);
4795                 }
4796
4797                 /* Wait a little for things to settle down */
4798                 udelay(50);
4799         }
4800         return 0;
4801 }
4802
4803 static uint64_t gfx_v11_0_get_gpu_clock_counter(struct amdgpu_device *adev)
4804 {
4805         uint64_t clock;
4806
4807         amdgpu_gfx_off_ctrl(adev, false);
4808         mutex_lock(&adev->gfx.gpu_clock_mutex);
4809         clock = (uint64_t)RREG32_SOC15(SMUIO, 0, regGOLDEN_TSC_COUNT_LOWER) |
4810                 ((uint64_t)RREG32_SOC15(SMUIO, 0, regGOLDEN_TSC_COUNT_UPPER) << 32ULL);
4811         mutex_unlock(&adev->gfx.gpu_clock_mutex);
4812         amdgpu_gfx_off_ctrl(adev, true);
4813         return clock;
4814 }
4815
4816 static void gfx_v11_0_ring_emit_gds_switch(struct amdgpu_ring *ring,
4817                                            uint32_t vmid,
4818                                            uint32_t gds_base, uint32_t gds_size,
4819                                            uint32_t gws_base, uint32_t gws_size,
4820                                            uint32_t oa_base, uint32_t oa_size)
4821 {
4822         struct amdgpu_device *adev = ring->adev;
4823
4824         /* GDS Base */
4825         gfx_v11_0_write_data_to_reg(ring, 0, false,
4826                                     SOC15_REG_OFFSET(GC, 0, regGDS_VMID0_BASE) + 2 * vmid,
4827                                     gds_base);
4828
4829         /* GDS Size */
4830         gfx_v11_0_write_data_to_reg(ring, 0, false,
4831                                     SOC15_REG_OFFSET(GC, 0, regGDS_VMID0_SIZE) + 2 * vmid,
4832                                     gds_size);
4833
4834         /* GWS */
4835         gfx_v11_0_write_data_to_reg(ring, 0, false,
4836                                     SOC15_REG_OFFSET(GC, 0, regGDS_GWS_VMID0) + vmid,
4837                                     gws_size << GDS_GWS_VMID0__SIZE__SHIFT | gws_base);
4838
4839         /* OA */
4840         gfx_v11_0_write_data_to_reg(ring, 0, false,
4841                                     SOC15_REG_OFFSET(GC, 0, regGDS_OA_VMID0) + vmid,
4842                                     (1 << (oa_size + oa_base)) - (1 << oa_base));
4843 }
4844
4845 static int gfx_v11_0_early_init(void *handle)
4846 {
4847         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4848
4849         adev->gfx.num_gfx_rings = GFX11_NUM_GFX_RINGS;
4850         adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev),
4851                                           AMDGPU_MAX_COMPUTE_RINGS);
4852
4853         gfx_v11_0_set_kiq_pm4_funcs(adev);
4854         gfx_v11_0_set_ring_funcs(adev);
4855         gfx_v11_0_set_irq_funcs(adev);
4856         gfx_v11_0_set_gds_init(adev);
4857         gfx_v11_0_set_rlc_funcs(adev);
4858         gfx_v11_0_set_mqd_funcs(adev);
4859         gfx_v11_0_set_imu_funcs(adev);
4860
4861         gfx_v11_0_init_rlcg_reg_access_ctrl(adev);
4862
4863         return 0;
4864 }
4865
4866 static int gfx_v11_0_late_init(void *handle)
4867 {
4868         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4869         int r;
4870
4871         r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0);
4872         if (r)
4873                 return r;
4874
4875         r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0);
4876         if (r)
4877                 return r;
4878
4879         return 0;
4880 }
4881
4882 static bool gfx_v11_0_is_rlc_enabled(struct amdgpu_device *adev)
4883 {
4884         uint32_t rlc_cntl;
4885
4886         /* if RLC is not enabled, do nothing */
4887         rlc_cntl = RREG32_SOC15(GC, 0, regRLC_CNTL);
4888         return (REG_GET_FIELD(rlc_cntl, RLC_CNTL, RLC_ENABLE_F32)) ? true : false;
4889 }
4890
4891 static void gfx_v11_0_set_safe_mode(struct amdgpu_device *adev)
4892 {
4893         uint32_t data;
4894         unsigned i;
4895
4896         data = RLC_SAFE_MODE__CMD_MASK;
4897         data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT);
4898
4899         WREG32_SOC15(GC, 0, regRLC_SAFE_MODE, data);
4900
4901         /* wait for RLC_SAFE_MODE */
4902         for (i = 0; i < adev->usec_timeout; i++) {
4903                 if (!REG_GET_FIELD(RREG32_SOC15(GC, 0, regRLC_SAFE_MODE),
4904                                    RLC_SAFE_MODE, CMD))
4905                         break;
4906                 udelay(1);
4907         }
4908 }
4909
4910 static void gfx_v11_0_unset_safe_mode(struct amdgpu_device *adev)
4911 {
4912         WREG32_SOC15(GC, 0, regRLC_SAFE_MODE, RLC_SAFE_MODE__CMD_MASK);
4913 }
4914
4915 static void gfx_v11_0_update_perf_clk(struct amdgpu_device *adev,
4916                                       bool enable)
4917 {
4918         uint32_t def, data;
4919
4920         if (!(adev->cg_flags & AMD_CG_SUPPORT_GFX_PERF_CLK))
4921                 return;
4922
4923         def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
4924
4925         if (enable)
4926                 data &= ~RLC_CGTT_MGCG_OVERRIDE__PERFMON_CLOCK_STATE_MASK;
4927         else
4928                 data |= RLC_CGTT_MGCG_OVERRIDE__PERFMON_CLOCK_STATE_MASK;
4929
4930         if (def != data)
4931                 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data);
4932 }
4933
4934 static void gfx_v11_0_update_sram_fgcg(struct amdgpu_device *adev,
4935                                        bool enable)
4936 {
4937         uint32_t def, data;
4938
4939         if (!(adev->cg_flags & AMD_CG_SUPPORT_GFX_FGCG))
4940                 return;
4941
4942         def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
4943
4944         if (enable)
4945                 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_FGCG_OVERRIDE_MASK;
4946         else
4947                 data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_FGCG_OVERRIDE_MASK;
4948
4949         if (def != data)
4950                 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data);
4951 }
4952
4953 static void gfx_v11_0_update_repeater_fgcg(struct amdgpu_device *adev,
4954                                            bool enable)
4955 {
4956         uint32_t def, data;
4957
4958         if (!(adev->cg_flags & AMD_CG_SUPPORT_REPEATER_FGCG))
4959                 return;
4960
4961         def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
4962
4963         if (enable)
4964                 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_REPEATER_FGCG_OVERRIDE_MASK;
4965         else
4966                 data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_REPEATER_FGCG_OVERRIDE_MASK;
4967
4968         if (def != data)
4969                 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data);
4970 }
4971
4972 static void gfx_v11_0_update_medium_grain_clock_gating(struct amdgpu_device *adev,
4973                                                        bool enable)
4974 {
4975         uint32_t data, def;
4976
4977         if (!(adev->cg_flags & (AMD_CG_SUPPORT_GFX_MGCG | AMD_CG_SUPPORT_GFX_MGLS)))
4978                 return;
4979
4980         /* It is disabled by HW by default */
4981         if (enable) {
4982                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG) {
4983                         /* 1 - RLC_CGTT_MGCG_OVERRIDE */
4984                         def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
4985
4986                         data &= ~(RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
4987                                   RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK |
4988                                   RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK);
4989
4990                         if (def != data)
4991                                 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data);
4992                 }
4993         } else {
4994                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG) {
4995                         def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
4996
4997                         data |= (RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK |
4998                                  RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
4999                                  RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK);
5000
5001                         if (def != data)
5002                                 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data);
5003                 }
5004         }
5005 }
5006
5007 static void gfx_v11_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev,
5008                                                        bool enable)
5009 {
5010         uint32_t def, data;
5011
5012         if (!(adev->cg_flags &
5013               (AMD_CG_SUPPORT_GFX_CGCG |
5014               AMD_CG_SUPPORT_GFX_CGLS |
5015               AMD_CG_SUPPORT_GFX_3D_CGCG |
5016               AMD_CG_SUPPORT_GFX_3D_CGLS)))
5017                 return;
5018
5019         if (enable) {
5020                 def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
5021
5022                 /* unset CGCG override */
5023                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)
5024                         data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGCG_OVERRIDE_MASK;
5025                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
5026                         data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK;
5027                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG ||
5028                     adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS)
5029                         data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_GFX3D_CG_OVERRIDE_MASK;
5030
5031                 /* update CGCG override bits */
5032                 if (def != data)
5033                         WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data);
5034
5035                 /* enable cgcg FSM(0x0000363F) */
5036                 def = data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL);
5037
5038                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG) {
5039                         data &= ~RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD_MASK;
5040                         data |= (0x36 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
5041                                  RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
5042                 }
5043
5044                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) {
5045                         data &= ~RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY_MASK;
5046                         data |= (0x000F << RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
5047                                  RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;
5048                 }
5049
5050                 if (def != data)
5051                         WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL, data);
5052
5053                 /* Program RLC_CGCG_CGLS_CTRL_3D */
5054                 def = data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D);
5055
5056                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG) {
5057                         data &= ~RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD_MASK;
5058                         data |= (0x36 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
5059                                  RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK;
5060                 }
5061
5062                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS) {
5063                         data &= ~RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY_MASK;
5064                         data |= (0xf << RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
5065                                  RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK;
5066                 }
5067
5068                 if (def != data)
5069                         WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D, data);
5070
5071                 /* set IDLE_POLL_COUNT(0x00900100) */
5072                 def = data = RREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_CNTL);
5073
5074                 data &= ~(CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY_MASK | CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK);
5075                 data |= (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) |
5076                         (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
5077
5078                 if (def != data)
5079                         WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_CNTL, data);
5080
5081                 data = RREG32_SOC15(GC, 0, regCP_INT_CNTL);
5082                 data = REG_SET_FIELD(data, CP_INT_CNTL, CNTX_BUSY_INT_ENABLE, 1);
5083                 data = REG_SET_FIELD(data, CP_INT_CNTL, CNTX_EMPTY_INT_ENABLE, 1);
5084                 data = REG_SET_FIELD(data, CP_INT_CNTL, CMP_BUSY_INT_ENABLE, 1);
5085                 data = REG_SET_FIELD(data, CP_INT_CNTL, GFX_IDLE_INT_ENABLE, 1);
5086                 WREG32_SOC15(GC, 0, regCP_INT_CNTL, data);
5087
5088                 data = RREG32_SOC15(GC, 0, regSDMA0_RLC_CGCG_CTRL);
5089                 data = REG_SET_FIELD(data, SDMA0_RLC_CGCG_CTRL, CGCG_INT_ENABLE, 1);
5090                 WREG32_SOC15(GC, 0, regSDMA0_RLC_CGCG_CTRL, data);
5091
5092                 data = RREG32_SOC15(GC, 0, regSDMA1_RLC_CGCG_CTRL);
5093                 data = REG_SET_FIELD(data, SDMA1_RLC_CGCG_CTRL, CGCG_INT_ENABLE, 1);
5094                 WREG32_SOC15(GC, 0, regSDMA1_RLC_CGCG_CTRL, data);
5095         } else {
5096                 /* Program RLC_CGCG_CGLS_CTRL */
5097                 def = data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL);
5098
5099                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)
5100                         data &= ~RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
5101
5102                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
5103                         data &= ~RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;
5104
5105                 if (def != data)
5106                         WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL, data);
5107
5108                 /* Program RLC_CGCG_CGLS_CTRL_3D */
5109                 def = data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D);
5110
5111                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG)
5112                         data &= ~RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK;
5113                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS)
5114                         data &= ~RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK;
5115
5116                 if (def != data)
5117                         WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D, data);
5118
5119                 data = RREG32_SOC15(GC, 0, regSDMA0_RLC_CGCG_CTRL);
5120                 data &= ~SDMA0_RLC_CGCG_CTRL__CGCG_INT_ENABLE_MASK;
5121                 WREG32_SOC15(GC, 0, regSDMA0_RLC_CGCG_CTRL, data);
5122
5123                 data = RREG32_SOC15(GC, 0, regSDMA1_RLC_CGCG_CTRL);
5124                 data &= ~SDMA1_RLC_CGCG_CTRL__CGCG_INT_ENABLE_MASK;
5125                 WREG32_SOC15(GC, 0, regSDMA1_RLC_CGCG_CTRL, data);
5126         }
5127 }
5128
5129 static int gfx_v11_0_update_gfx_clock_gating(struct amdgpu_device *adev,
5130                                             bool enable)
5131 {
5132         amdgpu_gfx_rlc_enter_safe_mode(adev);
5133
5134         gfx_v11_0_update_coarse_grain_clock_gating(adev, enable);
5135
5136         gfx_v11_0_update_medium_grain_clock_gating(adev, enable);
5137
5138         gfx_v11_0_update_repeater_fgcg(adev, enable);
5139
5140         gfx_v11_0_update_sram_fgcg(adev, enable);
5141
5142         gfx_v11_0_update_perf_clk(adev, enable);
5143
5144         if (adev->cg_flags &
5145             (AMD_CG_SUPPORT_GFX_MGCG |
5146              AMD_CG_SUPPORT_GFX_CGLS |
5147              AMD_CG_SUPPORT_GFX_CGCG |
5148              AMD_CG_SUPPORT_GFX_3D_CGCG |
5149              AMD_CG_SUPPORT_GFX_3D_CGLS))
5150                 gfx_v11_0_enable_gui_idle_interrupt(adev, enable);
5151
5152         amdgpu_gfx_rlc_exit_safe_mode(adev);
5153
5154         return 0;
5155 }
5156
5157 static void gfx_v11_0_update_spm_vmid(struct amdgpu_device *adev, unsigned vmid)
5158 {
5159         u32 reg, data;
5160
5161         reg = SOC15_REG_OFFSET(GC, 0, regRLC_SPM_MC_CNTL);
5162         if (amdgpu_sriov_is_pp_one_vf(adev))
5163                 data = RREG32_NO_KIQ(reg);
5164         else
5165                 data = RREG32(reg);
5166
5167         data &= ~RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK;
5168         data |= (vmid & RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK) << RLC_SPM_MC_CNTL__RLC_SPM_VMID__SHIFT;
5169
5170         if (amdgpu_sriov_is_pp_one_vf(adev))
5171                 WREG32_SOC15_NO_KIQ(GC, 0, regRLC_SPM_MC_CNTL, data);
5172         else
5173                 WREG32_SOC15(GC, 0, regRLC_SPM_MC_CNTL, data);
5174 }
5175
5176 static const struct amdgpu_rlc_funcs gfx_v11_0_rlc_funcs = {
5177         .is_rlc_enabled = gfx_v11_0_is_rlc_enabled,
5178         .set_safe_mode = gfx_v11_0_set_safe_mode,
5179         .unset_safe_mode = gfx_v11_0_unset_safe_mode,
5180         .init = gfx_v11_0_rlc_init,
5181         .get_csb_size = gfx_v11_0_get_csb_size,
5182         .get_csb_buffer = gfx_v11_0_get_csb_buffer,
5183         .resume = gfx_v11_0_rlc_resume,
5184         .stop = gfx_v11_0_rlc_stop,
5185         .reset = gfx_v11_0_rlc_reset,
5186         .start = gfx_v11_0_rlc_start,
5187         .update_spm_vmid = gfx_v11_0_update_spm_vmid,
5188 };
5189
5190 static int gfx_v11_0_set_powergating_state(void *handle,
5191                                            enum amd_powergating_state state)
5192 {
5193         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5194         bool enable = (state == AMD_PG_STATE_GATE);
5195
5196         if (amdgpu_sriov_vf(adev))
5197                 return 0;
5198
5199         switch (adev->ip_versions[GC_HWIP][0]) {
5200         case IP_VERSION(11, 0, 0):
5201         case IP_VERSION(11, 0, 2):
5202                 amdgpu_gfx_off_ctrl(adev, enable);
5203                 break;
5204         default:
5205                 break;
5206         }
5207
5208         return 0;
5209 }
5210
5211 static int gfx_v11_0_set_clockgating_state(void *handle,
5212                                           enum amd_clockgating_state state)
5213 {
5214         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5215
5216         if (amdgpu_sriov_vf(adev))
5217                 return 0;
5218
5219         switch (adev->ip_versions[GC_HWIP][0]) {
5220         case IP_VERSION(11, 0, 0):
5221         case IP_VERSION(11, 0, 2):
5222                 gfx_v11_0_update_gfx_clock_gating(adev,
5223                                 state ==  AMD_CG_STATE_GATE);
5224                 break;
5225         default:
5226                 break;
5227         }
5228
5229         return 0;
5230 }
5231
5232 static void gfx_v11_0_get_clockgating_state(void *handle, u64 *flags)
5233 {
5234         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5235         int data;
5236
5237         /* AMD_CG_SUPPORT_GFX_MGCG */
5238         data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
5239         if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK))
5240                 *flags |= AMD_CG_SUPPORT_GFX_MGCG;
5241
5242         /* AMD_CG_SUPPORT_REPEATER_FGCG */
5243         if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_REPEATER_FGCG_OVERRIDE_MASK))
5244                 *flags |= AMD_CG_SUPPORT_REPEATER_FGCG;
5245
5246         /* AMD_CG_SUPPORT_GFX_FGCG */
5247         if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_FGCG_OVERRIDE_MASK))
5248                 *flags |= AMD_CG_SUPPORT_GFX_FGCG;
5249
5250         /* AMD_CG_SUPPORT_GFX_PERF_CLK */
5251         if (!(data & RLC_CGTT_MGCG_OVERRIDE__PERFMON_CLOCK_STATE_MASK))
5252                 *flags |= AMD_CG_SUPPORT_GFX_PERF_CLK;
5253
5254         /* AMD_CG_SUPPORT_GFX_CGCG */
5255         data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL);
5256         if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK)
5257                 *flags |= AMD_CG_SUPPORT_GFX_CGCG;
5258
5259         /* AMD_CG_SUPPORT_GFX_CGLS */
5260         if (data & RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK)
5261                 *flags |= AMD_CG_SUPPORT_GFX_CGLS;
5262
5263         /* AMD_CG_SUPPORT_GFX_3D_CGCG */
5264         data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D);
5265         if (data & RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK)
5266                 *flags |= AMD_CG_SUPPORT_GFX_3D_CGCG;
5267
5268         /* AMD_CG_SUPPORT_GFX_3D_CGLS */
5269         if (data & RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK)
5270                 *flags |= AMD_CG_SUPPORT_GFX_3D_CGLS;
5271 }
5272
5273 static u64 gfx_v11_0_ring_get_rptr_gfx(struct amdgpu_ring *ring)
5274 {
5275         /* gfx11 is 32bit rptr*/
5276         return *(uint32_t *)ring->rptr_cpu_addr;
5277 }
5278
5279 static u64 gfx_v11_0_ring_get_wptr_gfx(struct amdgpu_ring *ring)
5280 {
5281         struct amdgpu_device *adev = ring->adev;
5282         u64 wptr;
5283
5284         /* XXX check if swapping is necessary on BE */
5285         if (ring->use_doorbell) {
5286                 wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr);
5287         } else {
5288                 wptr = RREG32_SOC15(GC, 0, regCP_RB0_WPTR);
5289                 wptr += (u64)RREG32_SOC15(GC, 0, regCP_RB0_WPTR_HI) << 32;
5290         }
5291
5292         return wptr;
5293 }
5294
5295 static void gfx_v11_0_ring_set_wptr_gfx(struct amdgpu_ring *ring)
5296 {
5297         struct amdgpu_device *adev = ring->adev;
5298
5299         if (ring->use_doorbell) {
5300                 /* XXX check if swapping is necessary on BE */
5301                 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, ring->wptr);
5302                 WDOORBELL64(ring->doorbell_index, ring->wptr);
5303         } else {
5304                 WREG32_SOC15(GC, 0, regCP_RB0_WPTR, lower_32_bits(ring->wptr));
5305                 WREG32_SOC15(GC, 0, regCP_RB0_WPTR_HI, upper_32_bits(ring->wptr));
5306         }
5307 }
5308
5309 static u64 gfx_v11_0_ring_get_rptr_compute(struct amdgpu_ring *ring)
5310 {
5311         /* gfx11 hardware is 32bit rptr */
5312         return *(uint32_t *)ring->rptr_cpu_addr;
5313 }
5314
5315 static u64 gfx_v11_0_ring_get_wptr_compute(struct amdgpu_ring *ring)
5316 {
5317         u64 wptr;
5318
5319         /* XXX check if swapping is necessary on BE */
5320         if (ring->use_doorbell)
5321                 wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr);
5322         else
5323                 BUG();
5324         return wptr;
5325 }
5326
5327 static void gfx_v11_0_ring_set_wptr_compute(struct amdgpu_ring *ring)
5328 {
5329         struct amdgpu_device *adev = ring->adev;
5330
5331         /* XXX check if swapping is necessary on BE */
5332         if (ring->use_doorbell) {
5333                 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, ring->wptr);
5334                 WDOORBELL64(ring->doorbell_index, ring->wptr);
5335         } else {
5336                 BUG(); /* only DOORBELL method supported on gfx11 now */
5337         }
5338 }
5339
5340 static void gfx_v11_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
5341 {
5342         struct amdgpu_device *adev = ring->adev;
5343         u32 ref_and_mask, reg_mem_engine;
5344         const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg;
5345
5346         if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
5347                 switch (ring->me) {
5348                 case 1:
5349                         ref_and_mask = nbio_hf_reg->ref_and_mask_cp2 << ring->pipe;
5350                         break;
5351                 case 2:
5352                         ref_and_mask = nbio_hf_reg->ref_and_mask_cp6 << ring->pipe;
5353                         break;
5354                 default:
5355                         return;
5356                 }
5357                 reg_mem_engine = 0;
5358         } else {
5359                 ref_and_mask = nbio_hf_reg->ref_and_mask_cp0;
5360                 reg_mem_engine = 1; /* pfp */
5361         }
5362
5363         gfx_v11_0_wait_reg_mem(ring, reg_mem_engine, 0, 1,
5364                                adev->nbio.funcs->get_hdp_flush_req_offset(adev),
5365                                adev->nbio.funcs->get_hdp_flush_done_offset(adev),
5366                                ref_and_mask, ref_and_mask, 0x20);
5367 }
5368
5369 static void gfx_v11_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
5370                                        struct amdgpu_job *job,
5371                                        struct amdgpu_ib *ib,
5372                                        uint32_t flags)
5373 {
5374         unsigned vmid = AMDGPU_JOB_GET_VMID(job);
5375         u32 header, control = 0;
5376
5377         BUG_ON(ib->flags & AMDGPU_IB_FLAG_CE);
5378
5379         header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
5380
5381         control |= ib->length_dw | (vmid << 24);
5382
5383         if ((amdgpu_sriov_vf(ring->adev) || amdgpu_mcbp) && (ib->flags & AMDGPU_IB_FLAG_PREEMPT)) {
5384                 control |= INDIRECT_BUFFER_PRE_ENB(1);
5385
5386                 if (flags & AMDGPU_IB_PREEMPTED)
5387                         control |= INDIRECT_BUFFER_PRE_RESUME(1);
5388
5389                 if (vmid)
5390                         gfx_v11_0_ring_emit_de_meta(ring,
5391                                     (!amdgpu_sriov_vf(ring->adev) && flags & AMDGPU_IB_PREEMPTED) ? true : false);
5392         }
5393
5394         if (ring->is_mes_queue)
5395                 /* inherit vmid from mqd */
5396                 control |= 0x400000;
5397
5398         amdgpu_ring_write(ring, header);
5399         BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
5400         amdgpu_ring_write(ring,
5401 #ifdef __BIG_ENDIAN
5402                 (2 << 0) |
5403 #endif
5404                 lower_32_bits(ib->gpu_addr));
5405         amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
5406         amdgpu_ring_write(ring, control);
5407 }
5408
5409 static void gfx_v11_0_ring_emit_ib_compute(struct amdgpu_ring *ring,
5410                                            struct amdgpu_job *job,
5411                                            struct amdgpu_ib *ib,
5412                                            uint32_t flags)
5413 {
5414         unsigned vmid = AMDGPU_JOB_GET_VMID(job);
5415         u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24);
5416
5417         if (ring->is_mes_queue)
5418                 /* inherit vmid from mqd */
5419                 control |= 0x40000000;
5420
5421         /* Currently, there is a high possibility to get wave ID mismatch
5422          * between ME and GDS, leading to a hw deadlock, because ME generates
5423          * different wave IDs than the GDS expects. This situation happens
5424          * randomly when at least 5 compute pipes use GDS ordered append.
5425          * The wave IDs generated by ME are also wrong after suspend/resume.
5426          * Those are probably bugs somewhere else in the kernel driver.
5427          *
5428          * Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and
5429          * GDS to 0 for this ring (me/pipe).
5430          */
5431         if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) {
5432                 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
5433                 amdgpu_ring_write(ring, regGDS_COMPUTE_MAX_WAVE_ID);
5434                 amdgpu_ring_write(ring, ring->adev->gds.gds_compute_max_wave_id);
5435         }
5436
5437         amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
5438         BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
5439         amdgpu_ring_write(ring,
5440 #ifdef __BIG_ENDIAN
5441                                 (2 << 0) |
5442 #endif
5443                                 lower_32_bits(ib->gpu_addr));
5444         amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
5445         amdgpu_ring_write(ring, control);
5446 }
5447
5448 static void gfx_v11_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
5449                                      u64 seq, unsigned flags)
5450 {
5451         bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
5452         bool int_sel = flags & AMDGPU_FENCE_FLAG_INT;
5453
5454         /* RELEASE_MEM - flush caches, send int */
5455         amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 6));
5456         amdgpu_ring_write(ring, (PACKET3_RELEASE_MEM_GCR_SEQ |
5457                                  PACKET3_RELEASE_MEM_GCR_GL2_WB |
5458                                  PACKET3_RELEASE_MEM_GCR_GL2_INV |
5459                                  PACKET3_RELEASE_MEM_GCR_GL2_US |
5460                                  PACKET3_RELEASE_MEM_GCR_GL1_INV |
5461                                  PACKET3_RELEASE_MEM_GCR_GLV_INV |
5462                                  PACKET3_RELEASE_MEM_GCR_GLM_INV |
5463                                  PACKET3_RELEASE_MEM_GCR_GLM_WB |
5464                                  PACKET3_RELEASE_MEM_CACHE_POLICY(3) |
5465                                  PACKET3_RELEASE_MEM_EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
5466                                  PACKET3_RELEASE_MEM_EVENT_INDEX(5)));
5467         amdgpu_ring_write(ring, (PACKET3_RELEASE_MEM_DATA_SEL(write64bit ? 2 : 1) |
5468                                  PACKET3_RELEASE_MEM_INT_SEL(int_sel ? 2 : 0)));
5469
5470         /*
5471          * the address should be Qword aligned if 64bit write, Dword
5472          * aligned if only send 32bit data low (discard data high)
5473          */
5474         if (write64bit)
5475                 BUG_ON(addr & 0x7);
5476         else
5477                 BUG_ON(addr & 0x3);
5478         amdgpu_ring_write(ring, lower_32_bits(addr));
5479         amdgpu_ring_write(ring, upper_32_bits(addr));
5480         amdgpu_ring_write(ring, lower_32_bits(seq));
5481         amdgpu_ring_write(ring, upper_32_bits(seq));
5482         amdgpu_ring_write(ring, ring->is_mes_queue ?
5483                          (ring->hw_queue_id | AMDGPU_FENCE_MES_QUEUE_FLAG) : 0);
5484 }
5485
5486 static void gfx_v11_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
5487 {
5488         int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
5489         uint32_t seq = ring->fence_drv.sync_seq;
5490         uint64_t addr = ring->fence_drv.gpu_addr;
5491
5492         gfx_v11_0_wait_reg_mem(ring, usepfp, 1, 0, lower_32_bits(addr),
5493                                upper_32_bits(addr), seq, 0xffffffff, 4);
5494 }
5495
5496 static void gfx_v11_0_ring_invalidate_tlbs(struct amdgpu_ring *ring,
5497                                    uint16_t pasid, uint32_t flush_type,
5498                                    bool all_hub, uint8_t dst_sel)
5499 {
5500         amdgpu_ring_write(ring, PACKET3(PACKET3_INVALIDATE_TLBS, 0));
5501         amdgpu_ring_write(ring,
5502                           PACKET3_INVALIDATE_TLBS_DST_SEL(dst_sel) |
5503                           PACKET3_INVALIDATE_TLBS_ALL_HUB(all_hub) |
5504                           PACKET3_INVALIDATE_TLBS_PASID(pasid) |
5505                           PACKET3_INVALIDATE_TLBS_FLUSH_TYPE(flush_type));
5506 }
5507
5508 static void gfx_v11_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
5509                                          unsigned vmid, uint64_t pd_addr)
5510 {
5511         if (ring->is_mes_queue)
5512                 gfx_v11_0_ring_invalidate_tlbs(ring, 0, 0, false, 0);
5513         else
5514                 amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
5515
5516         /* compute doesn't have PFP */
5517         if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) {
5518                 /* sync PFP to ME, otherwise we might get invalid PFP reads */
5519                 amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5520                 amdgpu_ring_write(ring, 0x0);
5521         }
5522 }
5523
5524 static void gfx_v11_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr,
5525                                           u64 seq, unsigned int flags)
5526 {
5527         struct amdgpu_device *adev = ring->adev;
5528
5529         /* we only allocate 32bit for each seq wb address */
5530         BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
5531
5532         /* write fence seq to the "addr" */
5533         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5534         amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5535                                  WRITE_DATA_DST_SEL(5) | WR_CONFIRM));
5536         amdgpu_ring_write(ring, lower_32_bits(addr));
5537         amdgpu_ring_write(ring, upper_32_bits(addr));
5538         amdgpu_ring_write(ring, lower_32_bits(seq));
5539
5540         if (flags & AMDGPU_FENCE_FLAG_INT) {
5541                 /* set register to trigger INT */
5542                 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5543                 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5544                                          WRITE_DATA_DST_SEL(0) | WR_CONFIRM));
5545                 amdgpu_ring_write(ring, SOC15_REG_OFFSET(GC, 0, regCPC_INT_STATUS));
5546                 amdgpu_ring_write(ring, 0);
5547                 amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */
5548         }
5549 }
5550
5551 static void gfx_v11_0_ring_emit_cntxcntl(struct amdgpu_ring *ring,
5552                                          uint32_t flags)
5553 {
5554         uint32_t dw2 = 0;
5555
5556         dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */
5557         if (flags & AMDGPU_HAVE_CTX_SWITCH) {
5558                 /* set load_global_config & load_global_uconfig */
5559                 dw2 |= 0x8001;
5560                 /* set load_cs_sh_regs */
5561                 dw2 |= 0x01000000;
5562                 /* set load_per_context_state & load_gfx_sh_regs for GFX */
5563                 dw2 |= 0x10002;
5564         }
5565
5566         amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
5567         amdgpu_ring_write(ring, dw2);
5568         amdgpu_ring_write(ring, 0);
5569 }
5570
5571 static unsigned gfx_v11_0_ring_emit_init_cond_exec(struct amdgpu_ring *ring)
5572 {
5573         unsigned ret;
5574
5575         amdgpu_ring_write(ring, PACKET3(PACKET3_COND_EXEC, 3));
5576         amdgpu_ring_write(ring, lower_32_bits(ring->cond_exe_gpu_addr));
5577         amdgpu_ring_write(ring, upper_32_bits(ring->cond_exe_gpu_addr));
5578         amdgpu_ring_write(ring, 0); /* discard following DWs if *cond_exec_gpu_addr==0 */
5579         ret = ring->wptr & ring->buf_mask;
5580         amdgpu_ring_write(ring, 0x55aa55aa); /* patch dummy value later */
5581
5582         return ret;
5583 }
5584
5585 static void gfx_v11_0_ring_emit_patch_cond_exec(struct amdgpu_ring *ring, unsigned offset)
5586 {
5587         unsigned cur;
5588         BUG_ON(offset > ring->buf_mask);
5589         BUG_ON(ring->ring[offset] != 0x55aa55aa);
5590
5591         cur = (ring->wptr - 1) & ring->buf_mask;
5592         if (likely(cur > offset))
5593                 ring->ring[offset] = cur - offset;
5594         else
5595                 ring->ring[offset] = (ring->buf_mask + 1) - offset + cur;
5596 }
5597
5598 static int gfx_v11_0_ring_preempt_ib(struct amdgpu_ring *ring)
5599 {
5600         int i, r = 0;
5601         struct amdgpu_device *adev = ring->adev;
5602         struct amdgpu_kiq *kiq = &adev->gfx.kiq;
5603         struct amdgpu_ring *kiq_ring = &kiq->ring;
5604         unsigned long flags;
5605
5606         if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues)
5607                 return -EINVAL;
5608
5609         spin_lock_irqsave(&kiq->ring_lock, flags);
5610
5611         if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size)) {
5612                 spin_unlock_irqrestore(&kiq->ring_lock, flags);
5613                 return -ENOMEM;
5614         }
5615
5616         /* assert preemption condition */
5617         amdgpu_ring_set_preempt_cond_exec(ring, false);
5618
5619         /* assert IB preemption, emit the trailing fence */
5620         kiq->pmf->kiq_unmap_queues(kiq_ring, ring, PREEMPT_QUEUES_NO_UNMAP,
5621                                    ring->trail_fence_gpu_addr,
5622                                    ++ring->trail_seq);
5623         amdgpu_ring_commit(kiq_ring);
5624
5625         spin_unlock_irqrestore(&kiq->ring_lock, flags);
5626
5627         /* poll the trailing fence */
5628         for (i = 0; i < adev->usec_timeout; i++) {
5629                 if (ring->trail_seq ==
5630                     le32_to_cpu(*(ring->trail_fence_cpu_addr)))
5631                         break;
5632                 udelay(1);
5633         }
5634
5635         if (i >= adev->usec_timeout) {
5636                 r = -EINVAL;
5637                 DRM_ERROR("ring %d failed to preempt ib\n", ring->idx);
5638         }
5639
5640         /* deassert preemption condition */
5641         amdgpu_ring_set_preempt_cond_exec(ring, true);
5642         return r;
5643 }
5644
5645 static void gfx_v11_0_ring_emit_de_meta(struct amdgpu_ring *ring, bool resume)
5646 {
5647         struct amdgpu_device *adev = ring->adev;
5648         struct v10_de_ib_state de_payload = {0};
5649         uint64_t offset, gds_addr, de_payload_gpu_addr;
5650         void *de_payload_cpu_addr;
5651         int cnt;
5652
5653         if (ring->is_mes_queue) {
5654                 offset = offsetof(struct amdgpu_mes_ctx_meta_data,
5655                                   gfx[0].gfx_meta_data) +
5656                         offsetof(struct v10_gfx_meta_data, de_payload);
5657                 de_payload_gpu_addr =
5658                         amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset);
5659                 de_payload_cpu_addr =
5660                         amdgpu_mes_ctx_get_offs_cpu_addr(ring, offset);
5661
5662                 offset = offsetof(struct amdgpu_mes_ctx_meta_data,
5663                                   gfx[0].gds_backup) +
5664                         offsetof(struct v10_gfx_meta_data, de_payload);
5665                 gds_addr = amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset);
5666         } else {
5667                 offset = offsetof(struct v10_gfx_meta_data, de_payload);
5668                 de_payload_gpu_addr = amdgpu_csa_vaddr(ring->adev) + offset;
5669                 de_payload_cpu_addr = adev->virt.csa_cpu_addr + offset;
5670
5671                 gds_addr = ALIGN(amdgpu_csa_vaddr(ring->adev) +
5672                                  AMDGPU_CSA_SIZE - adev->gds.gds_size,
5673                                  PAGE_SIZE);
5674         }
5675
5676         de_payload.gds_backup_addrlo = lower_32_bits(gds_addr);
5677         de_payload.gds_backup_addrhi = upper_32_bits(gds_addr);
5678
5679         cnt = (sizeof(de_payload) >> 2) + 4 - 2;
5680         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt));
5681         amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
5682                                  WRITE_DATA_DST_SEL(8) |
5683                                  WR_CONFIRM) |
5684                                  WRITE_DATA_CACHE_POLICY(0));
5685         amdgpu_ring_write(ring, lower_32_bits(de_payload_gpu_addr));
5686         amdgpu_ring_write(ring, upper_32_bits(de_payload_gpu_addr));
5687
5688         if (resume)
5689                 amdgpu_ring_write_multiple(ring, de_payload_cpu_addr,
5690                                            sizeof(de_payload) >> 2);
5691         else
5692                 amdgpu_ring_write_multiple(ring, (void *)&de_payload,
5693                                            sizeof(de_payload) >> 2);
5694 }
5695
5696 static void gfx_v11_0_ring_emit_frame_cntl(struct amdgpu_ring *ring, bool start,
5697                                     bool secure)
5698 {
5699         uint32_t v = secure ? FRAME_TMZ : 0;
5700
5701         amdgpu_ring_write(ring, PACKET3(PACKET3_FRAME_CONTROL, 0));
5702         amdgpu_ring_write(ring, v | FRAME_CMD(start ? 0 : 1));
5703 }
5704
5705 static void gfx_v11_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg,
5706                                      uint32_t reg_val_offs)
5707 {
5708         struct amdgpu_device *adev = ring->adev;
5709
5710         amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));
5711         amdgpu_ring_write(ring, 0 |     /* src: register*/
5712                                 (5 << 8) |      /* dst: memory */
5713                                 (1 << 20));     /* write confirm */
5714         amdgpu_ring_write(ring, reg);
5715         amdgpu_ring_write(ring, 0);
5716         amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +
5717                                 reg_val_offs * 4));
5718         amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +
5719                                 reg_val_offs * 4));
5720 }
5721
5722 static void gfx_v11_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg,
5723                                    uint32_t val)
5724 {
5725         uint32_t cmd = 0;
5726
5727         switch (ring->funcs->type) {
5728         case AMDGPU_RING_TYPE_GFX:
5729                 cmd = WRITE_DATA_ENGINE_SEL(1) | WR_CONFIRM;
5730                 break;
5731         case AMDGPU_RING_TYPE_KIQ:
5732                 cmd = (1 << 16); /* no inc addr */
5733                 break;
5734         default:
5735                 cmd = WR_CONFIRM;
5736                 break;
5737         }
5738         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5739         amdgpu_ring_write(ring, cmd);
5740         amdgpu_ring_write(ring, reg);
5741         amdgpu_ring_write(ring, 0);
5742         amdgpu_ring_write(ring, val);
5743 }
5744
5745 static void gfx_v11_0_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
5746                                         uint32_t val, uint32_t mask)
5747 {
5748         gfx_v11_0_wait_reg_mem(ring, 0, 0, 0, reg, 0, val, mask, 0x20);
5749 }
5750
5751 static void gfx_v11_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring,
5752                                                    uint32_t reg0, uint32_t reg1,
5753                                                    uint32_t ref, uint32_t mask)
5754 {
5755         int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
5756
5757         gfx_v11_0_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1,
5758                                ref, mask, 0x20);
5759 }
5760
5761 static void gfx_v11_0_ring_soft_recovery(struct amdgpu_ring *ring,
5762                                          unsigned vmid)
5763 {
5764         struct amdgpu_device *adev = ring->adev;
5765         uint32_t value = 0;
5766
5767         value = REG_SET_FIELD(value, SQ_CMD, CMD, 0x03);
5768         value = REG_SET_FIELD(value, SQ_CMD, MODE, 0x01);
5769         value = REG_SET_FIELD(value, SQ_CMD, CHECK_VMID, 1);
5770         value = REG_SET_FIELD(value, SQ_CMD, VM_ID, vmid);
5771         WREG32_SOC15(GC, 0, regSQ_CMD, value);
5772 }
5773
5774 static void
5775 gfx_v11_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev,
5776                                       uint32_t me, uint32_t pipe,
5777                                       enum amdgpu_interrupt_state state)
5778 {
5779         uint32_t cp_int_cntl, cp_int_cntl_reg;
5780
5781         if (!me) {
5782                 switch (pipe) {
5783                 case 0:
5784                         cp_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_INT_CNTL_RING0);
5785                         break;
5786                 case 1:
5787                         cp_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_INT_CNTL_RING1);
5788                         break;
5789                 default:
5790                         DRM_DEBUG("invalid pipe %d\n", pipe);
5791                         return;
5792                 }
5793         } else {
5794                 DRM_DEBUG("invalid me %d\n", me);
5795                 return;
5796         }
5797
5798         switch (state) {
5799         case AMDGPU_IRQ_STATE_DISABLE:
5800                 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg);
5801                 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
5802                                             TIME_STAMP_INT_ENABLE, 0);
5803                 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
5804                                             GENERIC0_INT_ENABLE, 0);
5805                 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl);
5806                 break;
5807         case AMDGPU_IRQ_STATE_ENABLE:
5808                 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg);
5809                 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
5810                                             TIME_STAMP_INT_ENABLE, 1);
5811                 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
5812                                             GENERIC0_INT_ENABLE, 1);
5813                 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl);
5814                 break;
5815         default:
5816                 break;
5817         }
5818 }
5819
5820 static void gfx_v11_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev,
5821                                                      int me, int pipe,
5822                                                      enum amdgpu_interrupt_state state)
5823 {
5824         u32 mec_int_cntl, mec_int_cntl_reg;
5825
5826         /*
5827          * amdgpu controls only the first MEC. That's why this function only
5828          * handles the setting of interrupts for this specific MEC. All other
5829          * pipes' interrupts are set by amdkfd.
5830          */
5831
5832         if (me == 1) {
5833                 switch (pipe) {
5834                 case 0:
5835                         mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE0_INT_CNTL);
5836                         break;
5837                 case 1:
5838                         mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE1_INT_CNTL);
5839                         break;
5840                 case 2:
5841                         mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE2_INT_CNTL);
5842                         break;
5843                 case 3:
5844                         mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE3_INT_CNTL);
5845                         break;
5846                 default:
5847                         DRM_DEBUG("invalid pipe %d\n", pipe);
5848                         return;
5849                 }
5850         } else {
5851                 DRM_DEBUG("invalid me %d\n", me);
5852                 return;
5853         }
5854
5855         switch (state) {
5856         case AMDGPU_IRQ_STATE_DISABLE:
5857                 mec_int_cntl = RREG32_SOC15_IP(GC, mec_int_cntl_reg);
5858                 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
5859                                              TIME_STAMP_INT_ENABLE, 0);
5860                 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
5861                                              GENERIC0_INT_ENABLE, 0);
5862                 WREG32_SOC15_IP(GC, mec_int_cntl_reg, mec_int_cntl);
5863                 break;
5864         case AMDGPU_IRQ_STATE_ENABLE:
5865                 mec_int_cntl = RREG32_SOC15_IP(GC, mec_int_cntl_reg);
5866                 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
5867                                              TIME_STAMP_INT_ENABLE, 1);
5868                 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
5869                                              GENERIC0_INT_ENABLE, 1);
5870                 WREG32_SOC15_IP(GC, mec_int_cntl_reg, mec_int_cntl);
5871                 break;
5872         default:
5873                 break;
5874         }
5875 }
5876
5877 static int gfx_v11_0_set_eop_interrupt_state(struct amdgpu_device *adev,
5878                                             struct amdgpu_irq_src *src,
5879                                             unsigned type,
5880                                             enum amdgpu_interrupt_state state)
5881 {
5882         switch (type) {
5883         case AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP:
5884                 gfx_v11_0_set_gfx_eop_interrupt_state(adev, 0, 0, state);
5885                 break;
5886         case AMDGPU_CP_IRQ_GFX_ME0_PIPE1_EOP:
5887                 gfx_v11_0_set_gfx_eop_interrupt_state(adev, 0, 1, state);
5888                 break;
5889         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP:
5890                 gfx_v11_0_set_compute_eop_interrupt_state(adev, 1, 0, state);
5891                 break;
5892         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP:
5893                 gfx_v11_0_set_compute_eop_interrupt_state(adev, 1, 1, state);
5894                 break;
5895         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP:
5896                 gfx_v11_0_set_compute_eop_interrupt_state(adev, 1, 2, state);
5897                 break;
5898         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP:
5899                 gfx_v11_0_set_compute_eop_interrupt_state(adev, 1, 3, state);
5900                 break;
5901         default:
5902                 break;
5903         }
5904         return 0;
5905 }
5906
5907 static int gfx_v11_0_eop_irq(struct amdgpu_device *adev,
5908                              struct amdgpu_irq_src *source,
5909                              struct amdgpu_iv_entry *entry)
5910 {
5911         int i;
5912         u8 me_id, pipe_id, queue_id;
5913         struct amdgpu_ring *ring;
5914         uint32_t mes_queue_id = entry->src_data[0];
5915
5916         DRM_DEBUG("IH: CP EOP\n");
5917
5918         if (adev->enable_mes && (mes_queue_id & AMDGPU_FENCE_MES_QUEUE_FLAG)) {
5919                 struct amdgpu_mes_queue *queue;
5920
5921                 mes_queue_id &= AMDGPU_FENCE_MES_QUEUE_ID_MASK;
5922
5923                 spin_lock(&adev->mes.queue_id_lock);
5924                 queue = idr_find(&adev->mes.queue_id_idr, mes_queue_id);
5925                 if (queue) {
5926                         DRM_DEBUG("process mes queue id = %d\n", mes_queue_id);
5927                         amdgpu_fence_process(queue->ring);
5928                 }
5929                 spin_unlock(&adev->mes.queue_id_lock);
5930         } else {
5931                 me_id = (entry->ring_id & 0x0c) >> 2;
5932                 pipe_id = (entry->ring_id & 0x03) >> 0;
5933                 queue_id = (entry->ring_id & 0x70) >> 4;
5934
5935                 switch (me_id) {
5936                 case 0:
5937                         if (pipe_id == 0)
5938                                 amdgpu_fence_process(&adev->gfx.gfx_ring[0]);
5939                         else
5940                                 amdgpu_fence_process(&adev->gfx.gfx_ring[1]);
5941                         break;
5942                 case 1:
5943                 case 2:
5944                         for (i = 0; i < adev->gfx.num_compute_rings; i++) {
5945                                 ring = &adev->gfx.compute_ring[i];
5946                                 /* Per-queue interrupt is supported for MEC starting from VI.
5947                                  * The interrupt can only be enabled/disabled per pipe instead
5948                                  * of per queue.
5949                                  */
5950                                 if ((ring->me == me_id) &&
5951                                     (ring->pipe == pipe_id) &&
5952                                     (ring->queue == queue_id))
5953                                         amdgpu_fence_process(ring);
5954                         }
5955                         break;
5956                 }
5957         }
5958
5959         return 0;
5960 }
5961
5962 static int gfx_v11_0_set_priv_reg_fault_state(struct amdgpu_device *adev,
5963                                               struct amdgpu_irq_src *source,
5964                                               unsigned type,
5965                                               enum amdgpu_interrupt_state state)
5966 {
5967         switch (state) {
5968         case AMDGPU_IRQ_STATE_DISABLE:
5969         case AMDGPU_IRQ_STATE_ENABLE:
5970                 WREG32_FIELD15_PREREG(GC, 0, CP_INT_CNTL_RING0,
5971                                PRIV_REG_INT_ENABLE,
5972                                state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
5973                 break;
5974         default:
5975                 break;
5976         }
5977
5978         return 0;
5979 }
5980
5981 static int gfx_v11_0_set_priv_inst_fault_state(struct amdgpu_device *adev,
5982                                                struct amdgpu_irq_src *source,
5983                                                unsigned type,
5984                                                enum amdgpu_interrupt_state state)
5985 {
5986         switch (state) {
5987         case AMDGPU_IRQ_STATE_DISABLE:
5988         case AMDGPU_IRQ_STATE_ENABLE:
5989                 WREG32_FIELD15_PREREG(GC, 0, CP_INT_CNTL_RING0,
5990                                PRIV_INSTR_INT_ENABLE,
5991                                state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
5992                 break;
5993         default:
5994                 break;
5995         }
5996
5997         return 0;
5998 }
5999
6000 static void gfx_v11_0_handle_priv_fault(struct amdgpu_device *adev,
6001                                         struct amdgpu_iv_entry *entry)
6002 {
6003         u8 me_id, pipe_id, queue_id;
6004         struct amdgpu_ring *ring;
6005         int i;
6006
6007         me_id = (entry->ring_id & 0x0c) >> 2;
6008         pipe_id = (entry->ring_id & 0x03) >> 0;
6009         queue_id = (entry->ring_id & 0x70) >> 4;
6010
6011         switch (me_id) {
6012         case 0:
6013                 for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
6014                         ring = &adev->gfx.gfx_ring[i];
6015                         /* we only enabled 1 gfx queue per pipe for now */
6016                         if (ring->me == me_id && ring->pipe == pipe_id)
6017                                 drm_sched_fault(&ring->sched);
6018                 }
6019                 break;
6020         case 1:
6021         case 2:
6022                 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
6023                         ring = &adev->gfx.compute_ring[i];
6024                         if (ring->me == me_id && ring->pipe == pipe_id &&
6025                             ring->queue == queue_id)
6026                                 drm_sched_fault(&ring->sched);
6027                 }
6028                 break;
6029         default:
6030                 BUG();
6031         }
6032 }
6033
6034 static int gfx_v11_0_priv_reg_irq(struct amdgpu_device *adev,
6035                                   struct amdgpu_irq_src *source,
6036                                   struct amdgpu_iv_entry *entry)
6037 {
6038         DRM_ERROR("Illegal register access in command stream\n");
6039         gfx_v11_0_handle_priv_fault(adev, entry);
6040         return 0;
6041 }
6042
6043 static int gfx_v11_0_priv_inst_irq(struct amdgpu_device *adev,
6044                                    struct amdgpu_irq_src *source,
6045                                    struct amdgpu_iv_entry *entry)
6046 {
6047         DRM_ERROR("Illegal instruction in command stream\n");
6048         gfx_v11_0_handle_priv_fault(adev, entry);
6049         return 0;
6050 }
6051
6052 #if 0
6053 static int gfx_v11_0_kiq_set_interrupt_state(struct amdgpu_device *adev,
6054                                              struct amdgpu_irq_src *src,
6055                                              unsigned int type,
6056                                              enum amdgpu_interrupt_state state)
6057 {
6058         uint32_t tmp, target;
6059         struct amdgpu_ring *ring = &(adev->gfx.kiq.ring);
6060
6061         target = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE0_INT_CNTL);
6062         target += ring->pipe;
6063
6064         switch (type) {
6065         case AMDGPU_CP_KIQ_IRQ_DRIVER0:
6066                 if (state == AMDGPU_IRQ_STATE_DISABLE) {
6067                         tmp = RREG32_SOC15(GC, 0, regCPC_INT_CNTL);
6068                         tmp = REG_SET_FIELD(tmp, CPC_INT_CNTL,
6069                                             GENERIC2_INT_ENABLE, 0);
6070                         WREG32_SOC15(GC, 0, regCPC_INT_CNTL, tmp);
6071
6072                         tmp = RREG32_SOC15_IP(GC, target);
6073                         tmp = REG_SET_FIELD(tmp, CP_ME1_PIPE0_INT_CNTL,
6074                                             GENERIC2_INT_ENABLE, 0);
6075                         WREG32_SOC15_IP(GC, target, tmp);
6076                 } else {
6077                         tmp = RREG32_SOC15(GC, 0, regCPC_INT_CNTL);
6078                         tmp = REG_SET_FIELD(tmp, CPC_INT_CNTL,
6079                                             GENERIC2_INT_ENABLE, 1);
6080                         WREG32_SOC15(GC, 0, regCPC_INT_CNTL, tmp);
6081
6082                         tmp = RREG32_SOC15_IP(GC, target);
6083                         tmp = REG_SET_FIELD(tmp, CP_ME1_PIPE0_INT_CNTL,
6084                                             GENERIC2_INT_ENABLE, 1);
6085                         WREG32_SOC15_IP(GC, target, tmp);
6086                 }
6087                 break;
6088         default:
6089                 BUG(); /* kiq only support GENERIC2_INT now */
6090                 break;
6091         }
6092         return 0;
6093 }
6094 #endif
6095
6096 static void gfx_v11_0_emit_mem_sync(struct amdgpu_ring *ring)
6097 {
6098         const unsigned int gcr_cntl =
6099                         PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_INV(1) |
6100                         PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_WB(1) |
6101                         PACKET3_ACQUIRE_MEM_GCR_CNTL_GLM_INV(1) |
6102                         PACKET3_ACQUIRE_MEM_GCR_CNTL_GLM_WB(1) |
6103                         PACKET3_ACQUIRE_MEM_GCR_CNTL_GL1_INV(1) |
6104                         PACKET3_ACQUIRE_MEM_GCR_CNTL_GLV_INV(1) |
6105                         PACKET3_ACQUIRE_MEM_GCR_CNTL_GLK_INV(1) |
6106                         PACKET3_ACQUIRE_MEM_GCR_CNTL_GLI_INV(1);
6107
6108         /* ACQUIRE_MEM - make one or more surfaces valid for use by the subsequent operations */
6109         amdgpu_ring_write(ring, PACKET3(PACKET3_ACQUIRE_MEM, 6));
6110         amdgpu_ring_write(ring, 0); /* CP_COHER_CNTL */
6111         amdgpu_ring_write(ring, 0xffffffff);  /* CP_COHER_SIZE */
6112         amdgpu_ring_write(ring, 0xffffff);  /* CP_COHER_SIZE_HI */
6113         amdgpu_ring_write(ring, 0); /* CP_COHER_BASE */
6114         amdgpu_ring_write(ring, 0);  /* CP_COHER_BASE_HI */
6115         amdgpu_ring_write(ring, 0x0000000A); /* POLL_INTERVAL */
6116         amdgpu_ring_write(ring, gcr_cntl); /* GCR_CNTL */
6117 }
6118
6119 static const struct amd_ip_funcs gfx_v11_0_ip_funcs = {
6120         .name = "gfx_v11_0",
6121         .early_init = gfx_v11_0_early_init,
6122         .late_init = gfx_v11_0_late_init,
6123         .sw_init = gfx_v11_0_sw_init,
6124         .sw_fini = gfx_v11_0_sw_fini,
6125         .hw_init = gfx_v11_0_hw_init,
6126         .hw_fini = gfx_v11_0_hw_fini,
6127         .suspend = gfx_v11_0_suspend,
6128         .resume = gfx_v11_0_resume,
6129         .is_idle = gfx_v11_0_is_idle,
6130         .wait_for_idle = gfx_v11_0_wait_for_idle,
6131         .soft_reset = gfx_v11_0_soft_reset,
6132         .set_clockgating_state = gfx_v11_0_set_clockgating_state,
6133         .set_powergating_state = gfx_v11_0_set_powergating_state,
6134         .get_clockgating_state = gfx_v11_0_get_clockgating_state,
6135 };
6136
6137 static const struct amdgpu_ring_funcs gfx_v11_0_ring_funcs_gfx = {
6138         .type = AMDGPU_RING_TYPE_GFX,
6139         .align_mask = 0xff,
6140         .nop = PACKET3(PACKET3_NOP, 0x3FFF),
6141         .support_64bit_ptrs = true,
6142         .vmhub = AMDGPU_GFXHUB_0,
6143         .get_rptr = gfx_v11_0_ring_get_rptr_gfx,
6144         .get_wptr = gfx_v11_0_ring_get_wptr_gfx,
6145         .set_wptr = gfx_v11_0_ring_set_wptr_gfx,
6146         .emit_frame_size = /* totally 242 maximum if 16 IBs */
6147                 5 + /* COND_EXEC */
6148                 7 + /* PIPELINE_SYNC */
6149                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
6150                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
6151                 2 + /* VM_FLUSH */
6152                 8 + /* FENCE for VM_FLUSH */
6153                 20 + /* GDS switch */
6154                 5 + /* COND_EXEC */
6155                 7 + /* HDP_flush */
6156                 4 + /* VGT_flush */
6157                 31 + /* DE_META */
6158                 3 + /* CNTX_CTRL */
6159                 5 + /* HDP_INVL */
6160                 8 + 8 + /* FENCE x2 */
6161                 8, /* gfx_v11_0_emit_mem_sync */
6162         .emit_ib_size = 4, /* gfx_v11_0_ring_emit_ib_gfx */
6163         .emit_ib = gfx_v11_0_ring_emit_ib_gfx,
6164         .emit_fence = gfx_v11_0_ring_emit_fence,
6165         .emit_pipeline_sync = gfx_v11_0_ring_emit_pipeline_sync,
6166         .emit_vm_flush = gfx_v11_0_ring_emit_vm_flush,
6167         .emit_gds_switch = gfx_v11_0_ring_emit_gds_switch,
6168         .emit_hdp_flush = gfx_v11_0_ring_emit_hdp_flush,
6169         .test_ring = gfx_v11_0_ring_test_ring,
6170         .test_ib = gfx_v11_0_ring_test_ib,
6171         .insert_nop = amdgpu_ring_insert_nop,
6172         .pad_ib = amdgpu_ring_generic_pad_ib,
6173         .emit_cntxcntl = gfx_v11_0_ring_emit_cntxcntl,
6174         .init_cond_exec = gfx_v11_0_ring_emit_init_cond_exec,
6175         .patch_cond_exec = gfx_v11_0_ring_emit_patch_cond_exec,
6176         .preempt_ib = gfx_v11_0_ring_preempt_ib,
6177         .emit_frame_cntl = gfx_v11_0_ring_emit_frame_cntl,
6178         .emit_wreg = gfx_v11_0_ring_emit_wreg,
6179         .emit_reg_wait = gfx_v11_0_ring_emit_reg_wait,
6180         .emit_reg_write_reg_wait = gfx_v11_0_ring_emit_reg_write_reg_wait,
6181         .soft_recovery = gfx_v11_0_ring_soft_recovery,
6182         .emit_mem_sync = gfx_v11_0_emit_mem_sync,
6183 };
6184
6185 static const struct amdgpu_ring_funcs gfx_v11_0_ring_funcs_compute = {
6186         .type = AMDGPU_RING_TYPE_COMPUTE,
6187         .align_mask = 0xff,
6188         .nop = PACKET3(PACKET3_NOP, 0x3FFF),
6189         .support_64bit_ptrs = true,
6190         .vmhub = AMDGPU_GFXHUB_0,
6191         .get_rptr = gfx_v11_0_ring_get_rptr_compute,
6192         .get_wptr = gfx_v11_0_ring_get_wptr_compute,
6193         .set_wptr = gfx_v11_0_ring_set_wptr_compute,
6194         .emit_frame_size =
6195                 20 + /* gfx_v11_0_ring_emit_gds_switch */
6196                 7 + /* gfx_v11_0_ring_emit_hdp_flush */
6197                 5 + /* hdp invalidate */
6198                 7 + /* gfx_v11_0_ring_emit_pipeline_sync */
6199                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
6200                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
6201                 2 + /* gfx_v11_0_ring_emit_vm_flush */
6202                 8 + 8 + 8 + /* gfx_v11_0_ring_emit_fence x3 for user fence, vm fence */
6203                 8, /* gfx_v11_0_emit_mem_sync */
6204         .emit_ib_size = 7, /* gfx_v11_0_ring_emit_ib_compute */
6205         .emit_ib = gfx_v11_0_ring_emit_ib_compute,
6206         .emit_fence = gfx_v11_0_ring_emit_fence,
6207         .emit_pipeline_sync = gfx_v11_0_ring_emit_pipeline_sync,
6208         .emit_vm_flush = gfx_v11_0_ring_emit_vm_flush,
6209         .emit_gds_switch = gfx_v11_0_ring_emit_gds_switch,
6210         .emit_hdp_flush = gfx_v11_0_ring_emit_hdp_flush,
6211         .test_ring = gfx_v11_0_ring_test_ring,
6212         .test_ib = gfx_v11_0_ring_test_ib,
6213         .insert_nop = amdgpu_ring_insert_nop,
6214         .pad_ib = amdgpu_ring_generic_pad_ib,
6215         .emit_wreg = gfx_v11_0_ring_emit_wreg,
6216         .emit_reg_wait = gfx_v11_0_ring_emit_reg_wait,
6217         .emit_reg_write_reg_wait = gfx_v11_0_ring_emit_reg_write_reg_wait,
6218         .emit_mem_sync = gfx_v11_0_emit_mem_sync,
6219 };
6220
6221 static const struct amdgpu_ring_funcs gfx_v11_0_ring_funcs_kiq = {
6222         .type = AMDGPU_RING_TYPE_KIQ,
6223         .align_mask = 0xff,
6224         .nop = PACKET3(PACKET3_NOP, 0x3FFF),
6225         .support_64bit_ptrs = true,
6226         .vmhub = AMDGPU_GFXHUB_0,
6227         .get_rptr = gfx_v11_0_ring_get_rptr_compute,
6228         .get_wptr = gfx_v11_0_ring_get_wptr_compute,
6229         .set_wptr = gfx_v11_0_ring_set_wptr_compute,
6230         .emit_frame_size =
6231                 20 + /* gfx_v11_0_ring_emit_gds_switch */
6232                 7 + /* gfx_v11_0_ring_emit_hdp_flush */
6233                 5 + /*hdp invalidate */
6234                 7 + /* gfx_v11_0_ring_emit_pipeline_sync */
6235                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
6236                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
6237                 2 + /* gfx_v11_0_ring_emit_vm_flush */
6238                 8 + 8 + 8, /* gfx_v11_0_ring_emit_fence_kiq x3 for user fence, vm fence */
6239         .emit_ib_size = 7, /* gfx_v11_0_ring_emit_ib_compute */
6240         .emit_ib = gfx_v11_0_ring_emit_ib_compute,
6241         .emit_fence = gfx_v11_0_ring_emit_fence_kiq,
6242         .test_ring = gfx_v11_0_ring_test_ring,
6243         .test_ib = gfx_v11_0_ring_test_ib,
6244         .insert_nop = amdgpu_ring_insert_nop,
6245         .pad_ib = amdgpu_ring_generic_pad_ib,
6246         .emit_rreg = gfx_v11_0_ring_emit_rreg,
6247         .emit_wreg = gfx_v11_0_ring_emit_wreg,
6248         .emit_reg_wait = gfx_v11_0_ring_emit_reg_wait,
6249         .emit_reg_write_reg_wait = gfx_v11_0_ring_emit_reg_write_reg_wait,
6250 };
6251
6252 static void gfx_v11_0_set_ring_funcs(struct amdgpu_device *adev)
6253 {
6254         int i;
6255
6256         adev->gfx.kiq.ring.funcs = &gfx_v11_0_ring_funcs_kiq;
6257
6258         for (i = 0; i < adev->gfx.num_gfx_rings; i++)
6259                 adev->gfx.gfx_ring[i].funcs = &gfx_v11_0_ring_funcs_gfx;
6260
6261         for (i = 0; i < adev->gfx.num_compute_rings; i++)
6262                 adev->gfx.compute_ring[i].funcs = &gfx_v11_0_ring_funcs_compute;
6263 }
6264
6265 static const struct amdgpu_irq_src_funcs gfx_v11_0_eop_irq_funcs = {
6266         .set = gfx_v11_0_set_eop_interrupt_state,
6267         .process = gfx_v11_0_eop_irq,
6268 };
6269
6270 static const struct amdgpu_irq_src_funcs gfx_v11_0_priv_reg_irq_funcs = {
6271         .set = gfx_v11_0_set_priv_reg_fault_state,
6272         .process = gfx_v11_0_priv_reg_irq,
6273 };
6274
6275 static const struct amdgpu_irq_src_funcs gfx_v11_0_priv_inst_irq_funcs = {
6276         .set = gfx_v11_0_set_priv_inst_fault_state,
6277         .process = gfx_v11_0_priv_inst_irq,
6278 };
6279
6280 static void gfx_v11_0_set_irq_funcs(struct amdgpu_device *adev)
6281 {
6282         adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST;
6283         adev->gfx.eop_irq.funcs = &gfx_v11_0_eop_irq_funcs;
6284
6285         adev->gfx.priv_reg_irq.num_types = 1;
6286         adev->gfx.priv_reg_irq.funcs = &gfx_v11_0_priv_reg_irq_funcs;
6287
6288         adev->gfx.priv_inst_irq.num_types = 1;
6289         adev->gfx.priv_inst_irq.funcs = &gfx_v11_0_priv_inst_irq_funcs;
6290 }
6291
6292 static void gfx_v11_0_set_imu_funcs(struct amdgpu_device *adev)
6293 {
6294         adev->gfx.imu.funcs = &gfx_v11_0_imu_funcs;
6295 }
6296
6297 static void gfx_v11_0_set_rlc_funcs(struct amdgpu_device *adev)
6298 {
6299         adev->gfx.rlc.funcs = &gfx_v11_0_rlc_funcs;
6300 }
6301
6302 static void gfx_v11_0_set_gds_init(struct amdgpu_device *adev)
6303 {
6304         unsigned total_cu = adev->gfx.config.max_cu_per_sh *
6305                             adev->gfx.config.max_sh_per_se *
6306                             adev->gfx.config.max_shader_engines;
6307
6308         adev->gds.gds_size = 0x1000;
6309         adev->gds.gds_compute_max_wave_id = total_cu * 32 - 1;
6310         adev->gds.gws_size = 64;
6311         adev->gds.oa_size = 16;
6312 }
6313
6314 static void gfx_v11_0_set_mqd_funcs(struct amdgpu_device *adev)
6315 {
6316         /* set gfx eng mqd */
6317         adev->mqds[AMDGPU_HW_IP_GFX].mqd_size =
6318                 sizeof(struct v11_gfx_mqd);
6319         adev->mqds[AMDGPU_HW_IP_GFX].init_mqd =
6320                 gfx_v11_0_gfx_mqd_init;
6321         /* set compute eng mqd */
6322         adev->mqds[AMDGPU_HW_IP_COMPUTE].mqd_size =
6323                 sizeof(struct v11_compute_mqd);
6324         adev->mqds[AMDGPU_HW_IP_COMPUTE].init_mqd =
6325                 gfx_v11_0_compute_mqd_init;
6326 }
6327
6328 static void gfx_v11_0_set_user_wgp_inactive_bitmap_per_sh(struct amdgpu_device *adev,
6329                                                           u32 bitmap)
6330 {
6331         u32 data;
6332
6333         if (!bitmap)
6334                 return;
6335
6336         data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_WGPS__SHIFT;
6337         data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_WGPS_MASK;
6338
6339         WREG32_SOC15(GC, 0, regGC_USER_SHADER_ARRAY_CONFIG, data);
6340 }
6341
6342 static u32 gfx_v11_0_get_wgp_active_bitmap_per_sh(struct amdgpu_device *adev)
6343 {
6344         u32 data, wgp_bitmask;
6345         data = RREG32_SOC15(GC, 0, regCC_GC_SHADER_ARRAY_CONFIG);
6346         data |= RREG32_SOC15(GC, 0, regGC_USER_SHADER_ARRAY_CONFIG);
6347
6348         data &= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_WGPS_MASK;
6349         data >>= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_WGPS__SHIFT;
6350
6351         wgp_bitmask =
6352                 amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh >> 1);
6353
6354         return (~data) & wgp_bitmask;
6355 }
6356
6357 static u32 gfx_v11_0_get_cu_active_bitmap_per_sh(struct amdgpu_device *adev)
6358 {
6359         u32 wgp_idx, wgp_active_bitmap;
6360         u32 cu_bitmap_per_wgp, cu_active_bitmap;
6361
6362         wgp_active_bitmap = gfx_v11_0_get_wgp_active_bitmap_per_sh(adev);
6363         cu_active_bitmap = 0;
6364
6365         for (wgp_idx = 0; wgp_idx < 16; wgp_idx++) {
6366                 /* if there is one WGP enabled, it means 2 CUs will be enabled */
6367                 cu_bitmap_per_wgp = 3 << (2 * wgp_idx);
6368                 if (wgp_active_bitmap & (1 << wgp_idx))
6369                         cu_active_bitmap |= cu_bitmap_per_wgp;
6370         }
6371
6372         return cu_active_bitmap;
6373 }
6374
6375 static int gfx_v11_0_get_cu_info(struct amdgpu_device *adev,
6376                                  struct amdgpu_cu_info *cu_info)
6377 {
6378         int i, j, k, counter, active_cu_number = 0;
6379         u32 mask, bitmap;
6380         unsigned disable_masks[8 * 2];
6381
6382         if (!adev || !cu_info)
6383                 return -EINVAL;
6384
6385         amdgpu_gfx_parse_disable_cu(disable_masks, 8, 2);
6386
6387         mutex_lock(&adev->grbm_idx_mutex);
6388         for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
6389                 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
6390                         mask = 1;
6391                         counter = 0;
6392                         gfx_v11_0_select_se_sh(adev, i, j, 0xffffffff);
6393                         if (i < 8 && j < 2)
6394                                 gfx_v11_0_set_user_wgp_inactive_bitmap_per_sh(
6395                                         adev, disable_masks[i * 2 + j]);
6396                         bitmap = gfx_v11_0_get_cu_active_bitmap_per_sh(adev);
6397
6398                         /**
6399                          * GFX11 could support more than 4 SEs, while the bitmap
6400                          * in cu_info struct is 4x4 and ioctl interface struct
6401                          * drm_amdgpu_info_device should keep stable.
6402                          * So we use last two columns of bitmap to store cu mask for
6403                          * SEs 4 to 7, the layout of the bitmap is as below:
6404                          *    SE0: {SH0,SH1} --> {bitmap[0][0], bitmap[0][1]}
6405                          *    SE1: {SH0,SH1} --> {bitmap[1][0], bitmap[1][1]}
6406                          *    SE2: {SH0,SH1} --> {bitmap[2][0], bitmap[2][1]}
6407                          *    SE3: {SH0,SH1} --> {bitmap[3][0], bitmap[3][1]}
6408                          *    SE4: {SH0,SH1} --> {bitmap[0][2], bitmap[0][3]}
6409                          *    SE5: {SH0,SH1} --> {bitmap[1][2], bitmap[1][3]}
6410                          *    SE6: {SH0,SH1} --> {bitmap[2][2], bitmap[2][3]}
6411                          *    SE7: {SH0,SH1} --> {bitmap[3][2], bitmap[3][3]}
6412                          */
6413                         cu_info->bitmap[i % 4][j + (i / 4) * 2] = bitmap;
6414
6415                         for (k = 0; k < adev->gfx.config.max_cu_per_sh; k++) {
6416                                 if (bitmap & mask)
6417                                         counter++;
6418
6419                                 mask <<= 1;
6420                         }
6421                         active_cu_number += counter;
6422                 }
6423         }
6424         gfx_v11_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
6425         mutex_unlock(&adev->grbm_idx_mutex);
6426
6427         cu_info->number = active_cu_number;
6428         cu_info->simd_per_cu = NUM_SIMD_PER_CU;
6429
6430         return 0;
6431 }
6432
6433 const struct amdgpu_ip_block_version gfx_v11_0_ip_block =
6434 {
6435         .type = AMD_IP_BLOCK_TYPE_GFX,
6436         .major = 11,
6437         .minor = 0,
6438         .rev = 0,
6439         .funcs = &gfx_v11_0_ip_funcs,
6440 };