drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c

   1 /*
   2  * Copyright 2021 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23 #include <linux/delay.h>
  24 #include <linux/kernel.h>
  25 #include <linux/firmware.h>
  26 #include <linux/module.h>
  27 #include <linux/pci.h>
  28 #include "amdgpu.h"
  29 #include "amdgpu_gfx.h"
  30 #include "amdgpu_psp.h"
  31 #include "amdgpu_smu.h"
  32 #include "amdgpu_atomfirmware.h"
  33 #include "imu_v11_0.h"
  34 #include "soc21.h"
  35 #include "nvd.h"
  36
  37 #include "gc/gc_11_0_0_offset.h"
  38 #include "gc/gc_11_0_0_sh_mask.h"
  39 #include "smuio/smuio_13_0_6_offset.h"
  40 #include "smuio/smuio_13_0_6_sh_mask.h"
  41 #include "navi10_enum.h"
  42 #include "ivsrcid/gfx/irqsrcs_gfx_11_0_0.h"
  43
  44 #include "soc15.h"
  45 #include "soc15d.h"
  46 #include "clearstate_gfx11.h"
  47 #include "v11_structs.h"
  48 #include "gfx_v11_0.h"
  49 #include "gfx_v11_0_3.h"
  50 #include "nbio_v4_3.h"
  51 #include "mes_v11_0.h"
  52
  53 #define GFX11_NUM_GFX_RINGS             1
  54 #define GFX11_MEC_HPD_SIZE      2048
  55
  56 #define RLCG_UCODE_LOADING_START_ADDRESS        0x00002000L
  57 #define RLC_PG_DELAY_3_DEFAULT_GC_11_0_1        0x1388
  58
  59 #define regCGTT_WD_CLK_CTRL             0x5086
  60 #define regCGTT_WD_CLK_CTRL_BASE_IDX    1
  61 #define regRLC_RLCS_BOOTLOAD_STATUS_gc_11_0_1   0x4e7e
  62 #define regRLC_RLCS_BOOTLOAD_STATUS_gc_11_0_1_BASE_IDX  1
  63 #define regPC_CONFIG_CNTL_1             0x194d
  64 #define regPC_CONFIG_CNTL_1_BASE_IDX    1
  65
  66 MODULE_FIRMWARE("amdgpu/gc_11_0_0_pfp.bin");
  67 MODULE_FIRMWARE("amdgpu/gc_11_0_0_me.bin");
  68 MODULE_FIRMWARE("amdgpu/gc_11_0_0_mec.bin");
  69 MODULE_FIRMWARE("amdgpu/gc_11_0_0_rlc.bin");
  70 MODULE_FIRMWARE("amdgpu/gc_11_0_0_toc.bin");
  71 MODULE_FIRMWARE("amdgpu/gc_11_0_1_pfp.bin");
  72 MODULE_FIRMWARE("amdgpu/gc_11_0_1_me.bin");
  73 MODULE_FIRMWARE("amdgpu/gc_11_0_1_mec.bin");
  74 MODULE_FIRMWARE("amdgpu/gc_11_0_1_rlc.bin");
  75 MODULE_FIRMWARE("amdgpu/gc_11_0_2_pfp.bin");
  76 MODULE_FIRMWARE("amdgpu/gc_11_0_2_me.bin");
  77 MODULE_FIRMWARE("amdgpu/gc_11_0_2_mec.bin");
  78 MODULE_FIRMWARE("amdgpu/gc_11_0_2_rlc.bin");
  79 MODULE_FIRMWARE("amdgpu/gc_11_0_3_pfp.bin");
  80 MODULE_FIRMWARE("amdgpu/gc_11_0_3_me.bin");
  81 MODULE_FIRMWARE("amdgpu/gc_11_0_3_mec.bin");
  82 MODULE_FIRMWARE("amdgpu/gc_11_0_3_rlc.bin");
  83 MODULE_FIRMWARE("amdgpu/gc_11_0_4_pfp.bin");
  84 MODULE_FIRMWARE("amdgpu/gc_11_0_4_me.bin");
  85 MODULE_FIRMWARE("amdgpu/gc_11_0_4_mec.bin");
  86 MODULE_FIRMWARE("amdgpu/gc_11_0_4_rlc.bin");
  87 MODULE_FIRMWARE("amdgpu/gc_11_5_0_pfp.bin");
  88 MODULE_FIRMWARE("amdgpu/gc_11_5_0_me.bin");
  89 MODULE_FIRMWARE("amdgpu/gc_11_5_0_mec.bin");
  90 MODULE_FIRMWARE("amdgpu/gc_11_5_0_rlc.bin");
  91
  92 static const struct soc15_reg_golden golden_settings_gc_11_0_1[] =
  93 {
  94         SOC15_REG_GOLDEN_VALUE(GC, 0, regCGTT_GS_NGG_CLK_CTRL, 0x9fff8fff, 0x00000010),
  95         SOC15_REG_GOLDEN_VALUE(GC, 0, regCGTT_WD_CLK_CTRL, 0xffff8fff, 0x00000010),
  96         SOC15_REG_GOLDEN_VALUE(GC, 0, regCPF_GCR_CNTL, 0x0007ffff, 0x0000c200),
  97         SOC15_REG_GOLDEN_VALUE(GC, 0, regGL2C_CTRL3, 0xffff001b, 0x00f01988),
  98         SOC15_REG_GOLDEN_VALUE(GC, 0, regPA_CL_ENHANCE, 0xf0ffffff, 0x00880007),
  99         SOC15_REG_GOLDEN_VALUE(GC, 0, regPA_SC_ENHANCE_3, 0xfffffffd, 0x00000008),
 100         SOC15_REG_GOLDEN_VALUE(GC, 0, regPA_SC_VRS_SURFACE_CNTL_1, 0xfff891ff, 0x55480100),
 101         SOC15_REG_GOLDEN_VALUE(GC, 0, regTA_CNTL_AUX, 0xf7f7ffff, 0x01030000),
 102         SOC15_REG_GOLDEN_VALUE(GC, 0, regTCP_CNTL2, 0xfcffffff, 0x0000000a)
 103 };
 104
 105 static const struct soc15_reg_golden golden_settings_gc_11_5_0[] = {
 106         SOC15_REG_GOLDEN_VALUE(GC, 0, regDB_DEBUG5, 0xffffffff, 0x00000800),
 107         SOC15_REG_GOLDEN_VALUE(GC, 0, regGB_ADDR_CONFIG, 0x0c1807ff, 0x00000242),
 108         SOC15_REG_GOLDEN_VALUE(GC, 0, regGCR_GENERAL_CNTL, 0x1ff1ffff, 0x00000500),
 109         SOC15_REG_GOLDEN_VALUE(GC, 0, regGL2A_ADDR_MATCH_MASK, 0xffffffff, 0xfffffff3),
 110         SOC15_REG_GOLDEN_VALUE(GC, 0, regGL2C_ADDR_MATCH_MASK, 0xffffffff, 0xfffffff3),
 111         SOC15_REG_GOLDEN_VALUE(GC, 0, regGL2C_CTRL, 0xffffffff, 0xf37fff3f),
 112         SOC15_REG_GOLDEN_VALUE(GC, 0, regGL2C_CTRL3, 0xfffffffb, 0x00f40188),
 113         SOC15_REG_GOLDEN_VALUE(GC, 0, regGL2C_CTRL4, 0xf0ffffff, 0x8000b007),
 114         SOC15_REG_GOLDEN_VALUE(GC, 0, regPA_CL_ENHANCE, 0xf1ffffff, 0x00880007),
 115         SOC15_REG_GOLDEN_VALUE(GC, 0, regPC_CONFIG_CNTL_1, 0xffffffff, 0x00010000),
 116         SOC15_REG_GOLDEN_VALUE(GC, 0, regTA_CNTL_AUX, 0xf7f7ffff, 0x01030000),
 117         SOC15_REG_GOLDEN_VALUE(GC, 0, regTA_CNTL2, 0x007f0000, 0x00000000),
 118         SOC15_REG_GOLDEN_VALUE(GC, 0, regTCP_CNTL2, 0xffcfffff, 0x0000200a),
 119         SOC15_REG_GOLDEN_VALUE(GC, 0, regUTCL1_CTRL_2, 0xffffffff, 0x0000048f)
 120 };
 121
 122 #define DEFAULT_SH_MEM_CONFIG \
 123         ((SH_MEM_ADDRESS_MODE_64 << SH_MEM_CONFIG__ADDRESS_MODE__SHIFT) | \
 124          (SH_MEM_ALIGNMENT_MODE_UNALIGNED << SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT) | \
 125          (3 << SH_MEM_CONFIG__INITIAL_INST_PREFETCH__SHIFT))
 126
 127 static void gfx_v11_0_disable_gpa_mode(struct amdgpu_device *adev);
 128 static void gfx_v11_0_set_ring_funcs(struct amdgpu_device *adev);
 129 static void gfx_v11_0_set_irq_funcs(struct amdgpu_device *adev);
 130 static void gfx_v11_0_set_gds_init(struct amdgpu_device *adev);
 131 static void gfx_v11_0_set_rlc_funcs(struct amdgpu_device *adev);
 132 static void gfx_v11_0_set_mqd_funcs(struct amdgpu_device *adev);
 133 static void gfx_v11_0_set_imu_funcs(struct amdgpu_device *adev);
 134 static int gfx_v11_0_get_cu_info(struct amdgpu_device *adev,
 135                                  struct amdgpu_cu_info *cu_info);
 136 static uint64_t gfx_v11_0_get_gpu_clock_counter(struct amdgpu_device *adev);
 137 static void gfx_v11_0_select_se_sh(struct amdgpu_device *adev, u32 se_num,
 138                                    u32 sh_num, u32 instance, int xcc_id);
 139 static u32 gfx_v11_0_get_wgp_active_bitmap_per_sh(struct amdgpu_device *adev);
 140
 141 static void gfx_v11_0_ring_emit_de_meta(struct amdgpu_ring *ring, bool resume);
 142 static void gfx_v11_0_ring_emit_frame_cntl(struct amdgpu_ring *ring, bool start, bool secure);
 143 static void gfx_v11_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg,
 144                                      uint32_t val);
 145 static int gfx_v11_0_wait_for_rlc_autoload_complete(struct amdgpu_device *adev);
 146 static void gfx_v11_0_ring_invalidate_tlbs(struct amdgpu_ring *ring,
 147                                            uint16_t pasid, uint32_t flush_type,
 148                                            bool all_hub, uint8_t dst_sel);
 149 static void gfx_v11_0_set_safe_mode(struct amdgpu_device *adev, int xcc_id);
 150 static void gfx_v11_0_unset_safe_mode(struct amdgpu_device *adev, int xcc_id);
 151 static void gfx_v11_0_update_perf_clk(struct amdgpu_device *adev,
 152                                       bool enable);
 153
 154 static void gfx11_kiq_set_resources(struct amdgpu_ring *kiq_ring, uint64_t queue_mask)
 155 {
 156         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_RESOURCES, 6));
 157         amdgpu_ring_write(kiq_ring, PACKET3_SET_RESOURCES_VMID_MASK(0) |
 158                           PACKET3_SET_RESOURCES_QUEUE_TYPE(0)); /* vmid_mask:0 queue_type:0 (KIQ) */
 159         amdgpu_ring_write(kiq_ring, lower_32_bits(queue_mask)); /* queue mask lo */
 160         amdgpu_ring_write(kiq_ring, upper_32_bits(queue_mask)); /* queue mask hi */
 161         amdgpu_ring_write(kiq_ring, 0); /* gws mask lo */
 162         amdgpu_ring_write(kiq_ring, 0); /* gws mask hi */
 163         amdgpu_ring_write(kiq_ring, 0); /* oac mask */
 164         amdgpu_ring_write(kiq_ring, 0); /* gds heap base:0, gds heap size:0 */
 165 }
 166
 167 static void gfx11_kiq_map_queues(struct amdgpu_ring *kiq_ring,
 168                                  struct amdgpu_ring *ring)
 169 {
 170         uint64_t mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj);
 171         uint64_t wptr_addr = ring->wptr_gpu_addr;
 172         uint32_t me = 0, eng_sel = 0;
 173
 174         switch (ring->funcs->type) {
 175         case AMDGPU_RING_TYPE_COMPUTE:
 176                 me = 1;
 177                 eng_sel = 0;
 178                 break;
 179         case AMDGPU_RING_TYPE_GFX:
 180                 me = 0;
 181                 eng_sel = 4;
 182                 break;
 183         case AMDGPU_RING_TYPE_MES:
 184                 me = 2;
 185                 eng_sel = 5;
 186                 break;
 187         default:
 188                 WARN_ON(1);
 189         }
 190
 191         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5));
 192         /* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/
 193         amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
 194                           PACKET3_MAP_QUEUES_QUEUE_SEL(0) | /* Queue_Sel */
 195                           PACKET3_MAP_QUEUES_VMID(0) | /* VMID */
 196                           PACKET3_MAP_QUEUES_QUEUE(ring->queue) |
 197                           PACKET3_MAP_QUEUES_PIPE(ring->pipe) |
 198                           PACKET3_MAP_QUEUES_ME((me)) |
 199                           PACKET3_MAP_QUEUES_QUEUE_TYPE(0) | /*queue_type: normal compute queue */
 200                           PACKET3_MAP_QUEUES_ALLOC_FORMAT(0) | /* alloc format: all_on_one_pipe */
 201                           PACKET3_MAP_QUEUES_ENGINE_SEL(eng_sel) |
 202                           PACKET3_MAP_QUEUES_NUM_QUEUES(1)); /* num_queues: must be 1 */
 203         amdgpu_ring_write(kiq_ring, PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index));
 204         amdgpu_ring_write(kiq_ring, lower_32_bits(mqd_addr));
 205         amdgpu_ring_write(kiq_ring, upper_32_bits(mqd_addr));
 206         amdgpu_ring_write(kiq_ring, lower_32_bits(wptr_addr));
 207         amdgpu_ring_write(kiq_ring, upper_32_bits(wptr_addr));
 208 }
 209
 210 static void gfx11_kiq_unmap_queues(struct amdgpu_ring *kiq_ring,
 211                                    struct amdgpu_ring *ring,
 212                                    enum amdgpu_unmap_queues_action action,
 213                                    u64 gpu_addr, u64 seq)
 214 {
 215         struct amdgpu_device *adev = kiq_ring->adev;
 216         uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
 217
 218         if (adev->enable_mes && !adev->gfx.kiq[0].ring.sched.ready) {
 219                 amdgpu_mes_unmap_legacy_queue(adev, ring, action, gpu_addr, seq);
 220                 return;
 221         }
 222
 223         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4));
 224         amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
 225                           PACKET3_UNMAP_QUEUES_ACTION(action) |
 226                           PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) |
 227                           PACKET3_UNMAP_QUEUES_ENGINE_SEL(eng_sel) |
 228                           PACKET3_UNMAP_QUEUES_NUM_QUEUES(1));
 229         amdgpu_ring_write(kiq_ring,
 230                   PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index));
 231
 232         if (action == PREEMPT_QUEUES_NO_UNMAP) {
 233                 amdgpu_ring_write(kiq_ring, lower_32_bits(gpu_addr));
 234                 amdgpu_ring_write(kiq_ring, upper_32_bits(gpu_addr));
 235                 amdgpu_ring_write(kiq_ring, seq);
 236         } else {
 237                 amdgpu_ring_write(kiq_ring, 0);
 238                 amdgpu_ring_write(kiq_ring, 0);
 239                 amdgpu_ring_write(kiq_ring, 0);
 240         }
 241 }
 242
 243 static void gfx11_kiq_query_status(struct amdgpu_ring *kiq_ring,
 244                                    struct amdgpu_ring *ring,
 245                                    u64 addr,
 246                                    u64 seq)
 247 {
 248         uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
 249
 250         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_QUERY_STATUS, 5));
 251         amdgpu_ring_write(kiq_ring,
 252                           PACKET3_QUERY_STATUS_CONTEXT_ID(0) |
 253                           PACKET3_QUERY_STATUS_INTERRUPT_SEL(0) |
 254                           PACKET3_QUERY_STATUS_COMMAND(2));
 255         amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
 256                           PACKET3_QUERY_STATUS_DOORBELL_OFFSET(ring->doorbell_index) |
 257                           PACKET3_QUERY_STATUS_ENG_SEL(eng_sel));
 258         amdgpu_ring_write(kiq_ring, lower_32_bits(addr));
 259         amdgpu_ring_write(kiq_ring, upper_32_bits(addr));
 260         amdgpu_ring_write(kiq_ring, lower_32_bits(seq));
 261         amdgpu_ring_write(kiq_ring, upper_32_bits(seq));
 262 }
 263
 264 static void gfx11_kiq_invalidate_tlbs(struct amdgpu_ring *kiq_ring,
 265                                 uint16_t pasid, uint32_t flush_type,
 266                                 bool all_hub)
 267 {
 268         gfx_v11_0_ring_invalidate_tlbs(kiq_ring, pasid, flush_type, all_hub, 1);
 269 }
 270
 271 static const struct kiq_pm4_funcs gfx_v11_0_kiq_pm4_funcs = {
 272         .kiq_set_resources = gfx11_kiq_set_resources,
 273         .kiq_map_queues = gfx11_kiq_map_queues,
 274         .kiq_unmap_queues = gfx11_kiq_unmap_queues,
 275         .kiq_query_status = gfx11_kiq_query_status,
 276         .kiq_invalidate_tlbs = gfx11_kiq_invalidate_tlbs,
 277         .set_resources_size = 8,
 278         .map_queues_size = 7,
 279         .unmap_queues_size = 6,
 280         .query_status_size = 7,
 281         .invalidate_tlbs_size = 2,
 282 };
 283
 284 static void gfx_v11_0_set_kiq_pm4_funcs(struct amdgpu_device *adev)
 285 {
 286         adev->gfx.kiq[0].pmf = &gfx_v11_0_kiq_pm4_funcs;
 287 }
 288
 289 static void gfx_v11_0_init_golden_registers(struct amdgpu_device *adev)
 290 {
 291         switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
 292         case IP_VERSION(11, 0, 1):
 293         case IP_VERSION(11, 0, 4):
 294                 soc15_program_register_sequence(adev,
 295                                                 golden_settings_gc_11_0_1,
 296                                                 (const u32)ARRAY_SIZE(golden_settings_gc_11_0_1));
 297                 break;
 298         case IP_VERSION(11, 5, 0):
 299                 soc15_program_register_sequence(adev,
 300                                                 golden_settings_gc_11_5_0,
 301                                                 (const u32)ARRAY_SIZE(golden_settings_gc_11_5_0));
 302                 break;
 303         default:
 304                 break;
 305         }
 306 }
 307
 308 static void gfx_v11_0_write_data_to_reg(struct amdgpu_ring *ring, int eng_sel,
 309                                        bool wc, uint32_t reg, uint32_t val)
 310 {
 311         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
 312         amdgpu_ring_write(ring, WRITE_DATA_ENGINE_SEL(eng_sel) |
 313                           WRITE_DATA_DST_SEL(0) | (wc ? WR_CONFIRM : 0));
 314         amdgpu_ring_write(ring, reg);
 315         amdgpu_ring_write(ring, 0);
 316         amdgpu_ring_write(ring, val);
 317 }
 318
 319 static void gfx_v11_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel,
 320                                   int mem_space, int opt, uint32_t addr0,
 321                                   uint32_t addr1, uint32_t ref, uint32_t mask,
 322                                   uint32_t inv)
 323 {
 324         amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
 325         amdgpu_ring_write(ring,
 326                           /* memory (1) or register (0) */
 327                           (WAIT_REG_MEM_MEM_SPACE(mem_space) |
 328                            WAIT_REG_MEM_OPERATION(opt) | /* wait */
 329                            WAIT_REG_MEM_FUNCTION(3) |  /* equal */
 330                            WAIT_REG_MEM_ENGINE(eng_sel)));
 331
 332         if (mem_space)
 333                 BUG_ON(addr0 & 0x3); /* Dword align */
 334         amdgpu_ring_write(ring, addr0);
 335         amdgpu_ring_write(ring, addr1);
 336         amdgpu_ring_write(ring, ref);
 337         amdgpu_ring_write(ring, mask);
 338         amdgpu_ring_write(ring, inv); /* poll interval */
 339 }
 340
 341 static int gfx_v11_0_ring_test_ring(struct amdgpu_ring *ring)
 342 {
 343         struct amdgpu_device *adev = ring->adev;
 344         uint32_t scratch = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG0);
 345         uint32_t tmp = 0;
 346         unsigned i;
 347         int r;
 348
 349         WREG32(scratch, 0xCAFEDEAD);
 350         r = amdgpu_ring_alloc(ring, 5);
 351         if (r) {
 352                 DRM_ERROR("amdgpu: cp failed to lock ring %d (%d).\n",
 353                           ring->idx, r);
 354                 return r;
 355         }
 356
 357         if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ) {
 358                 gfx_v11_0_ring_emit_wreg(ring, scratch, 0xDEADBEEF);
 359         } else {
 360                 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
 361                 amdgpu_ring_write(ring, scratch -
 362                                   PACKET3_SET_UCONFIG_REG_START);
 363                 amdgpu_ring_write(ring, 0xDEADBEEF);
 364         }
 365         amdgpu_ring_commit(ring);
 366
 367         for (i = 0; i < adev->usec_timeout; i++) {
 368                 tmp = RREG32(scratch);
 369                 if (tmp == 0xDEADBEEF)
 370                         break;
 371                 if (amdgpu_emu_mode == 1)
 372                         msleep(1);
 373                 else
 374                         udelay(1);
 375         }
 376
 377         if (i >= adev->usec_timeout)
 378                 r = -ETIMEDOUT;
 379         return r;
 380 }
 381
 382 static int gfx_v11_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 383 {
 384         struct amdgpu_device *adev = ring->adev;
 385         struct amdgpu_ib ib;
 386         struct dma_fence *f = NULL;
 387         unsigned index;
 388         uint64_t gpu_addr;
 389         volatile uint32_t *cpu_ptr;
 390         long r;
 391
 392         /* MES KIQ fw hasn't indirect buffer support for now */
 393         if (adev->enable_mes_kiq &&
 394             ring->funcs->type == AMDGPU_RING_TYPE_KIQ)
 395                 return 0;
 396
 397         memset(&ib, 0, sizeof(ib));
 398
 399         if (ring->is_mes_queue) {
 400                 uint32_t padding, offset;
 401
 402                 offset = amdgpu_mes_ctx_get_offs(ring, AMDGPU_MES_CTX_IB_OFFS);
 403                 padding = amdgpu_mes_ctx_get_offs(ring,
 404                                                   AMDGPU_MES_CTX_PADDING_OFFS);
 405
 406                 ib.gpu_addr = amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset);
 407                 ib.ptr = amdgpu_mes_ctx_get_offs_cpu_addr(ring, offset);
 408
 409                 gpu_addr = amdgpu_mes_ctx_get_offs_gpu_addr(ring, padding);
 410                 cpu_ptr = amdgpu_mes_ctx_get_offs_cpu_addr(ring, padding);
 411                 *cpu_ptr = cpu_to_le32(0xCAFEDEAD);
 412         } else {
 413                 r = amdgpu_device_wb_get(adev, &index);
 414                 if (r)
 415                         return r;
 416
 417                 gpu_addr = adev->wb.gpu_addr + (index * 4);
 418                 adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD);
 419                 cpu_ptr = &adev->wb.wb[index];
 420
 421                 r = amdgpu_ib_get(adev, NULL, 16, AMDGPU_IB_POOL_DIRECT, &ib);
 422                 if (r) {
 423                         DRM_ERROR("amdgpu: failed to get ib (%ld).\n", r);
 424                         goto err1;
 425                 }
 426         }
 427
 428         ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3);
 429         ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM;
 430         ib.ptr[2] = lower_32_bits(gpu_addr);
 431         ib.ptr[3] = upper_32_bits(gpu_addr);
 432         ib.ptr[4] = 0xDEADBEEF;
 433         ib.length_dw = 5;
 434
 435         r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
 436         if (r)
 437                 goto err2;
 438
 439         r = dma_fence_wait_timeout(f, false, timeout);
 440         if (r == 0) {
 441                 r = -ETIMEDOUT;
 442                 goto err2;
 443         } else if (r < 0) {
 444                 goto err2;
 445         }
 446
 447         if (le32_to_cpu(*cpu_ptr) == 0xDEADBEEF)
 448                 r = 0;
 449         else
 450                 r = -EINVAL;
 451 err2:
 452         if (!ring->is_mes_queue)
 453                 amdgpu_ib_free(adev, &ib, NULL);
 454         dma_fence_put(f);
 455 err1:
 456         if (!ring->is_mes_queue)
 457                 amdgpu_device_wb_free(adev, index);
 458         return r;
 459 }
 460
 461 static void gfx_v11_0_free_microcode(struct amdgpu_device *adev)
 462 {
 463         amdgpu_ucode_release(&adev->gfx.pfp_fw);
 464         amdgpu_ucode_release(&adev->gfx.me_fw);
 465         amdgpu_ucode_release(&adev->gfx.rlc_fw);
 466         amdgpu_ucode_release(&adev->gfx.mec_fw);
 467
 468         kfree(adev->gfx.rlc.register_list_format);
 469 }
 470
 471 static int gfx_v11_0_init_toc_microcode(struct amdgpu_device *adev, const char *ucode_prefix)
 472 {
 473         const struct psp_firmware_header_v1_0 *toc_hdr;
 474         int err = 0;
 475         char fw_name[40];
 476
 477         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_toc.bin", ucode_prefix);
 478         err = amdgpu_ucode_request(adev, &adev->psp.toc_fw, fw_name);
 479         if (err)
 480                 goto out;
 481
 482         toc_hdr = (const struct psp_firmware_header_v1_0 *)adev->psp.toc_fw->data;
 483         adev->psp.toc.fw_version = le32_to_cpu(toc_hdr->header.ucode_version);
 484         adev->psp.toc.feature_version = le32_to_cpu(toc_hdr->sos.fw_version);
 485         adev->psp.toc.size_bytes = le32_to_cpu(toc_hdr->header.ucode_size_bytes);
 486         adev->psp.toc.start_addr = (uint8_t *)toc_hdr +
 487                                 le32_to_cpu(toc_hdr->header.ucode_array_offset_bytes);
 488         return 0;
 489 out:
 490         amdgpu_ucode_release(&adev->psp.toc_fw);
 491         return err;
 492 }
 493
 494 static void gfx_v11_0_check_fw_cp_gfx_shadow(struct amdgpu_device *adev)
 495 {
 496         switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
 497         case IP_VERSION(11, 0, 0):
 498         case IP_VERSION(11, 0, 2):
 499         case IP_VERSION(11, 0, 3):
 500                 if ((adev->gfx.me_fw_version >= 1505) &&
 501                     (adev->gfx.pfp_fw_version >= 1600) &&
 502                     (adev->gfx.mec_fw_version >= 512)) {
 503                         if (amdgpu_sriov_vf(adev))
 504                                 adev->gfx.cp_gfx_shadow = true;
 505                         else
 506                                 adev->gfx.cp_gfx_shadow = false;
 507                 }
 508                 break;
 509         default:
 510                 adev->gfx.cp_gfx_shadow = false;
 511                 break;
 512         }
 513 }
 514
 515 static int gfx_v11_0_init_microcode(struct amdgpu_device *adev)
 516 {
 517         char fw_name[40];
 518         char ucode_prefix[30];
 519         int err;
 520         const struct rlc_firmware_header_v2_0 *rlc_hdr;
 521         uint16_t version_major;
 522         uint16_t version_minor;
 523
 524         DRM_DEBUG("\n");
 525
 526         amdgpu_ucode_ip_version_decode(adev, GC_HWIP, ucode_prefix, sizeof(ucode_prefix));
 527
 528         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_pfp.bin", ucode_prefix);
 529         err = amdgpu_ucode_request(adev, &adev->gfx.pfp_fw, fw_name);
 530         if (err)
 531                 goto out;
 532         /* check pfp fw hdr version to decide if enable rs64 for gfx11.*/
 533         adev->gfx.rs64_enable = amdgpu_ucode_hdr_version(
 534                                 (union amdgpu_firmware_header *)
 535                                 adev->gfx.pfp_fw->data, 2, 0);
 536         if (adev->gfx.rs64_enable) {
 537                 dev_info(adev->dev, "CP RS64 enable\n");
 538                 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_PFP);
 539                 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_PFP_P0_STACK);
 540                 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_PFP_P1_STACK);
 541         } else {
 542                 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_PFP);
 543         }
 544
 545         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_me.bin", ucode_prefix);
 546         err = amdgpu_ucode_request(adev, &adev->gfx.me_fw, fw_name);
 547         if (err)
 548                 goto out;
 549         if (adev->gfx.rs64_enable) {
 550                 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_ME);
 551                 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_ME_P0_STACK);
 552                 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_ME_P1_STACK);
 553         } else {
 554                 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_ME);
 555         }
 556
 557         if (!amdgpu_sriov_vf(adev)) {
 558                 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_rlc.bin", ucode_prefix);
 559                 err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw, fw_name);
 560                 if (err)
 561                         goto out;
 562                 rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
 563                 version_major = le16_to_cpu(rlc_hdr->header.header_version_major);
 564                 version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor);
 565                 err = amdgpu_gfx_rlc_init_microcode(adev, version_major, version_minor);
 566                 if (err)
 567                         goto out;
 568         }
 569
 570         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec.bin", ucode_prefix);
 571         err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw, fw_name);
 572         if (err)
 573                 goto out;
 574         if (adev->gfx.rs64_enable) {
 575                 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC);
 576                 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P0_STACK);
 577                 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P1_STACK);
 578                 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P2_STACK);
 579                 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P3_STACK);
 580         } else {
 581                 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC1);
 582                 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC1_JT);
 583         }
 584
 585         if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO)
 586                 err = gfx_v11_0_init_toc_microcode(adev, ucode_prefix);
 587
 588         /* only one MEC for gfx 11.0.0. */
 589         adev->gfx.mec2_fw = NULL;
 590
 591         gfx_v11_0_check_fw_cp_gfx_shadow(adev);
 592 out:
 593         if (err) {
 594                 amdgpu_ucode_release(&adev->gfx.pfp_fw);
 595                 amdgpu_ucode_release(&adev->gfx.me_fw);
 596                 amdgpu_ucode_release(&adev->gfx.rlc_fw);
 597                 amdgpu_ucode_release(&adev->gfx.mec_fw);
 598         }
 599
 600         return err;
 601 }
 602
 603 static u32 gfx_v11_0_get_csb_size(struct amdgpu_device *adev)
 604 {
 605         u32 count = 0;
 606         const struct cs_section_def *sect = NULL;
 607         const struct cs_extent_def *ext = NULL;
 608
 609         /* begin clear state */
 610         count += 2;
 611         /* context control state */
 612         count += 3;
 613
 614         for (sect = gfx11_cs_data; sect->section != NULL; ++sect) {
 615                 for (ext = sect->section; ext->extent != NULL; ++ext) {
 616                         if (sect->id == SECT_CONTEXT)
 617                                 count += 2 + ext->reg_count;
 618                         else
 619                                 return 0;
 620                 }
 621         }
 622
 623         /* set PA_SC_TILE_STEERING_OVERRIDE */
 624         count += 3;
 625         /* end clear state */
 626         count += 2;
 627         /* clear state */
 628         count += 2;
 629
 630         return count;
 631 }
 632
 633 static void gfx_v11_0_get_csb_buffer(struct amdgpu_device *adev,
 634                                     volatile u32 *buffer)
 635 {
 636         u32 count = 0, i;
 637         const struct cs_section_def *sect = NULL;
 638         const struct cs_extent_def *ext = NULL;
 639         int ctx_reg_offset;
 640
 641         if (adev->gfx.rlc.cs_data == NULL)
 642                 return;
 643         if (buffer == NULL)
 644                 return;
 645
 646         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
 647         buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
 648
 649         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
 650         buffer[count++] = cpu_to_le32(0x80000000);
 651         buffer[count++] = cpu_to_le32(0x80000000);
 652
 653         for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) {
 654                 for (ext = sect->section; ext->extent != NULL; ++ext) {
 655                         if (sect->id == SECT_CONTEXT) {
 656                                 buffer[count++] =
 657                                         cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
 658                                 buffer[count++] = cpu_to_le32(ext->reg_index -
 659                                                 PACKET3_SET_CONTEXT_REG_START);
 660                                 for (i = 0; i < ext->reg_count; i++)
 661                                         buffer[count++] = cpu_to_le32(ext->extent[i]);
 662                         } else {
 663                                 return;
 664                         }
 665                 }
 666         }
 667
 668         ctx_reg_offset =
 669                 SOC15_REG_OFFSET(GC, 0, regPA_SC_TILE_STEERING_OVERRIDE) - PACKET3_SET_CONTEXT_REG_START;
 670         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 1));
 671         buffer[count++] = cpu_to_le32(ctx_reg_offset);
 672         buffer[count++] = cpu_to_le32(adev->gfx.config.pa_sc_tile_steering_override);
 673
 674         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
 675         buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
 676
 677         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
 678         buffer[count++] = cpu_to_le32(0);
 679 }
 680
 681 static void gfx_v11_0_rlc_fini(struct amdgpu_device *adev)
 682 {
 683         /* clear state block */
 684         amdgpu_bo_free_kernel(&adev->gfx.rlc.clear_state_obj,
 685                         &adev->gfx.rlc.clear_state_gpu_addr,
 686                         (void **)&adev->gfx.rlc.cs_ptr);
 687
 688         /* jump table block */
 689         amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj,
 690                         &adev->gfx.rlc.cp_table_gpu_addr,
 691                         (void **)&adev->gfx.rlc.cp_table_ptr);
 692 }
 693
 694 static void gfx_v11_0_init_rlcg_reg_access_ctrl(struct amdgpu_device *adev)
 695 {
 696         struct amdgpu_rlcg_reg_access_ctrl *reg_access_ctrl;
 697
 698         reg_access_ctrl = &adev->gfx.rlc.reg_access_ctrl[0];
 699         reg_access_ctrl->scratch_reg0 = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG0);
 700         reg_access_ctrl->scratch_reg1 = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG1);
 701         reg_access_ctrl->scratch_reg2 = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG2);
 702         reg_access_ctrl->scratch_reg3 = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG3);
 703         reg_access_ctrl->grbm_cntl = SOC15_REG_OFFSET(GC, 0, regGRBM_GFX_CNTL);
 704         reg_access_ctrl->grbm_idx = SOC15_REG_OFFSET(GC, 0, regGRBM_GFX_INDEX);
 705         reg_access_ctrl->spare_int = SOC15_REG_OFFSET(GC, 0, regRLC_SPARE_INT_0);
 706         adev->gfx.rlc.rlcg_reg_access_supported = true;
 707 }
 708
 709 static int gfx_v11_0_rlc_init(struct amdgpu_device *adev)
 710 {
 711         const struct cs_section_def *cs_data;
 712         int r;
 713
 714         adev->gfx.rlc.cs_data = gfx11_cs_data;
 715
 716         cs_data = adev->gfx.rlc.cs_data;
 717
 718         if (cs_data) {
 719                 /* init clear state block */
 720                 r = amdgpu_gfx_rlc_init_csb(adev);
 721                 if (r)
 722                         return r;
 723         }
 724
 725         /* init spm vmid with 0xf */
 726         if (adev->gfx.rlc.funcs->update_spm_vmid)
 727                 adev->gfx.rlc.funcs->update_spm_vmid(adev, 0xf);
 728
 729         return 0;
 730 }
 731
 732 static void gfx_v11_0_mec_fini(struct amdgpu_device *adev)
 733 {
 734         amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL);
 735         amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_obj, NULL, NULL);
 736         amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_data_obj, NULL, NULL);
 737 }
 738
 739 static void gfx_v11_0_me_init(struct amdgpu_device *adev)
 740 {
 741         bitmap_zero(adev->gfx.me.queue_bitmap, AMDGPU_MAX_GFX_QUEUES);
 742
 743         amdgpu_gfx_graphics_queue_acquire(adev);
 744 }
 745
 746 static int gfx_v11_0_mec_init(struct amdgpu_device *adev)
 747 {
 748         int r;
 749         u32 *hpd;
 750         size_t mec_hpd_size;
 751
 752         bitmap_zero(adev->gfx.mec_bitmap[0].queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
 753
 754         /* take ownership of the relevant compute queues */
 755         amdgpu_gfx_compute_queue_acquire(adev);
 756         mec_hpd_size = adev->gfx.num_compute_rings * GFX11_MEC_HPD_SIZE;
 757
 758         if (mec_hpd_size) {
 759                 r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE,
 760                                               AMDGPU_GEM_DOMAIN_GTT,
 761                                               &adev->gfx.mec.hpd_eop_obj,
 762                                               &adev->gfx.mec.hpd_eop_gpu_addr,
 763                                               (void **)&hpd);
 764                 if (r) {
 765                         dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r);
 766                         gfx_v11_0_mec_fini(adev);
 767                         return r;
 768                 }
 769
 770                 memset(hpd, 0, mec_hpd_size);
 771
 772                 amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj);
 773                 amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj);
 774         }
 775
 776         return 0;
 777 }
 778
 779 static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t wave, uint32_t address)
 780 {
 781         WREG32_SOC15(GC, 0, regSQ_IND_INDEX,
 782                 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
 783                 (address << SQ_IND_INDEX__INDEX__SHIFT));
 784         return RREG32_SOC15(GC, 0, regSQ_IND_DATA);
 785 }
 786
 787 static void wave_read_regs(struct amdgpu_device *adev, uint32_t wave,
 788                            uint32_t thread, uint32_t regno,
 789                            uint32_t num, uint32_t *out)
 790 {
 791         WREG32_SOC15(GC, 0, regSQ_IND_INDEX,
 792                 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
 793                 (regno << SQ_IND_INDEX__INDEX__SHIFT) |
 794                 (thread << SQ_IND_INDEX__WORKITEM_ID__SHIFT) |
 795                 (SQ_IND_INDEX__AUTO_INCR_MASK));
 796         while (num--)
 797                 *(out++) = RREG32_SOC15(GC, 0, regSQ_IND_DATA);
 798 }
 799
 800 static void gfx_v11_0_read_wave_data(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields)
 801 {
 802         /* in gfx11 the SIMD_ID is specified as part of the INSTANCE
 803          * field when performing a select_se_sh so it should be
 804          * zero here */
 805         WARN_ON(simd != 0);
 806
 807         /* type 3 wave data */
 808         dst[(*no_fields)++] = 3;
 809         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_STATUS);
 810         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_PC_LO);
 811         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_PC_HI);
 812         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_EXEC_LO);
 813         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_EXEC_HI);
 814         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_HW_ID1);
 815         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_HW_ID2);
 816         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_GPR_ALLOC);
 817         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_LDS_ALLOC);
 818         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_TRAPSTS);
 819         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_IB_STS);
 820         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_IB_STS2);
 821         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_IB_DBG1);
 822         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_M0);
 823         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_MODE);
 824 }
 825
 826 static void gfx_v11_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd,
 827                                      uint32_t wave, uint32_t start,
 828                                      uint32_t size, uint32_t *dst)
 829 {
 830         WARN_ON(simd != 0);
 831
 832         wave_read_regs(
 833                 adev, wave, 0, start + SQIND_WAVE_SGPRS_OFFSET, size,
 834                 dst);
 835 }
 836
 837 static void gfx_v11_0_read_wave_vgprs(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd,
 838                                       uint32_t wave, uint32_t thread,
 839                                       uint32_t start, uint32_t size,
 840                                       uint32_t *dst)
 841 {
 842         wave_read_regs(
 843                 adev, wave, thread,
 844                 start + SQIND_WAVE_VGPRS_OFFSET, size, dst);
 845 }
 846
 847 static void gfx_v11_0_select_me_pipe_q(struct amdgpu_device *adev,
 848                                         u32 me, u32 pipe, u32 q, u32 vm, u32 xcc_id)
 849 {
 850         soc21_grbm_select(adev, me, pipe, q, vm);
 851 }
 852
 853 /* all sizes are in bytes */
 854 #define MQD_SHADOW_BASE_SIZE      73728
 855 #define MQD_SHADOW_BASE_ALIGNMENT 256
 856 #define MQD_FWWORKAREA_SIZE       484
 857 #define MQD_FWWORKAREA_ALIGNMENT  256
 858
 859 static int gfx_v11_0_get_gfx_shadow_info(struct amdgpu_device *adev,
 860                                          struct amdgpu_gfx_shadow_info *shadow_info)
 861 {
 862         if (adev->gfx.cp_gfx_shadow) {
 863                 shadow_info->shadow_size = MQD_SHADOW_BASE_SIZE;
 864                 shadow_info->shadow_alignment = MQD_SHADOW_BASE_ALIGNMENT;
 865                 shadow_info->csa_size = MQD_FWWORKAREA_SIZE;
 866                 shadow_info->csa_alignment = MQD_FWWORKAREA_ALIGNMENT;
 867                 return 0;
 868         } else {
 869                 memset(shadow_info, 0, sizeof(struct amdgpu_gfx_shadow_info));
 870                 return -ENOTSUPP;
 871         }
 872 }
 873
 874 static const struct amdgpu_gfx_funcs gfx_v11_0_gfx_funcs = {
 875         .get_gpu_clock_counter = &gfx_v11_0_get_gpu_clock_counter,
 876         .select_se_sh = &gfx_v11_0_select_se_sh,
 877         .read_wave_data = &gfx_v11_0_read_wave_data,
 878         .read_wave_sgprs = &gfx_v11_0_read_wave_sgprs,
 879         .read_wave_vgprs = &gfx_v11_0_read_wave_vgprs,
 880         .select_me_pipe_q = &gfx_v11_0_select_me_pipe_q,
 881         .update_perfmon_mgcg = &gfx_v11_0_update_perf_clk,
 882         .get_gfx_shadow_info = &gfx_v11_0_get_gfx_shadow_info,
 883 };
 884
 885 static int gfx_v11_0_gpu_early_init(struct amdgpu_device *adev)
 886 {
 887         switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
 888         case IP_VERSION(11, 0, 0):
 889         case IP_VERSION(11, 0, 2):
 890                 adev->gfx.config.max_hw_contexts = 8;
 891                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
 892                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
 893                 adev->gfx.config.sc_hiz_tile_fifo_size = 0;
 894                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
 895                 break;
 896         case IP_VERSION(11, 0, 3):
 897                 adev->gfx.ras = &gfx_v11_0_3_ras;
 898                 adev->gfx.config.max_hw_contexts = 8;
 899                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
 900                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
 901                 adev->gfx.config.sc_hiz_tile_fifo_size = 0;
 902                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
 903                 break;
 904         case IP_VERSION(11, 0, 1):
 905         case IP_VERSION(11, 0, 4):
 906         case IP_VERSION(11, 5, 0):
 907                 adev->gfx.config.max_hw_contexts = 8;
 908                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
 909                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
 910                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x80;
 911                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x300;
 912                 break;
 913         default:
 914                 BUG();
 915                 break;
 916         }
 917
 918         return 0;
 919 }
 920
 921 static int gfx_v11_0_gfx_ring_init(struct amdgpu_device *adev, int ring_id,
 922                                    int me, int pipe, int queue)
 923 {
 924         int r;
 925         struct amdgpu_ring *ring;
 926         unsigned int irq_type;
 927
 928         ring = &adev->gfx.gfx_ring[ring_id];
 929
 930         ring->me = me;
 931         ring->pipe = pipe;
 932         ring->queue = queue;
 933
 934         ring->ring_obj = NULL;
 935         ring->use_doorbell = true;
 936
 937         if (!ring_id)
 938                 ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1;
 939         else
 940                 ring->doorbell_index = adev->doorbell_index.gfx_ring1 << 1;
 941         ring->vm_hub = AMDGPU_GFXHUB(0);
 942         sprintf(ring->name, "gfx_%d.%d.%d", ring->me, ring->pipe, ring->queue);
 943
 944         irq_type = AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP + ring->pipe;
 945         r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type,
 946                              AMDGPU_RING_PRIO_DEFAULT, NULL);
 947         if (r)
 948                 return r;
 949         return 0;
 950 }
 951
 952 static int gfx_v11_0_compute_ring_init(struct amdgpu_device *adev, int ring_id,
 953                                        int mec, int pipe, int queue)
 954 {
 955         int r;
 956         unsigned irq_type;
 957         struct amdgpu_ring *ring;
 958         unsigned int hw_prio;
 959
 960         ring = &adev->gfx.compute_ring[ring_id];
 961
 962         /* mec0 is me1 */
 963         ring->me = mec + 1;
 964         ring->pipe = pipe;
 965         ring->queue = queue;
 966
 967         ring->ring_obj = NULL;
 968         ring->use_doorbell = true;
 969         ring->doorbell_index = (adev->doorbell_index.mec_ring0 + ring_id) << 1;
 970         ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr
 971                                 + (ring_id * GFX11_MEC_HPD_SIZE);
 972         ring->vm_hub = AMDGPU_GFXHUB(0);
 973         sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue);
 974
 975         irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
 976                 + ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec)
 977                 + ring->pipe;
 978         hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring) ?
 979                         AMDGPU_GFX_PIPE_PRIO_HIGH : AMDGPU_GFX_PIPE_PRIO_NORMAL;
 980         /* type-2 packets are deprecated on MEC, use type-3 instead */
 981         r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type,
 982                              hw_prio, NULL);
 983         if (r)
 984                 return r;
 985
 986         return 0;
 987 }
 988
 989 static struct {
 990         SOC21_FIRMWARE_ID       id;
 991         unsigned int            offset;
 992         unsigned int            size;
 993 } rlc_autoload_info[SOC21_FIRMWARE_ID_MAX];
 994
 995 static void gfx_v11_0_parse_rlc_toc(struct amdgpu_device *adev, void *rlc_toc)
 996 {
 997         RLC_TABLE_OF_CONTENT *ucode = rlc_toc;
 998
 999         while (ucode && (ucode->id > SOC21_FIRMWARE_ID_INVALID) &&
1000                         (ucode->id < SOC21_FIRMWARE_ID_MAX)) {
1001                 rlc_autoload_info[ucode->id].id = ucode->id;
1002                 rlc_autoload_info[ucode->id].offset = ucode->offset * 4;
1003                 rlc_autoload_info[ucode->id].size = ucode->size * 4;
1004
1005                 ucode++;
1006         }
1007 }
1008
1009 static uint32_t gfx_v11_0_calc_toc_total_size(struct amdgpu_device *adev)
1010 {
1011         uint32_t total_size = 0;
1012         SOC21_FIRMWARE_ID id;
1013
1014         gfx_v11_0_parse_rlc_toc(adev, adev->psp.toc.start_addr);
1015
1016         for (id = SOC21_FIRMWARE_ID_RLC_G_UCODE; id < SOC21_FIRMWARE_ID_MAX; id++)
1017                 total_size += rlc_autoload_info[id].size;
1018
1019         /* In case the offset in rlc toc ucode is aligned */
1020         if (total_size < rlc_autoload_info[SOC21_FIRMWARE_ID_MAX-1].offset)
1021                 total_size = rlc_autoload_info[SOC21_FIRMWARE_ID_MAX-1].offset +
1022                         rlc_autoload_info[SOC21_FIRMWARE_ID_MAX-1].size;
1023
1024         return total_size;
1025 }
1026
1027 static int gfx_v11_0_rlc_autoload_buffer_init(struct amdgpu_device *adev)
1028 {
1029         int r;
1030         uint32_t total_size;
1031
1032         total_size = gfx_v11_0_calc_toc_total_size(adev);
1033
1034         r = amdgpu_bo_create_reserved(adev, total_size, 64 * 1024,
1035                                       AMDGPU_GEM_DOMAIN_VRAM |
1036                                       AMDGPU_GEM_DOMAIN_GTT,
1037                                       &adev->gfx.rlc.rlc_autoload_bo,
1038                                       &adev->gfx.rlc.rlc_autoload_gpu_addr,
1039                                       (void **)&adev->gfx.rlc.rlc_autoload_ptr);
1040
1041         if (r) {
1042                 dev_err(adev->dev, "(%d) failed to create fw autoload bo\n", r);
1043                 return r;
1044         }
1045
1046         return 0;
1047 }
1048
1049 static void gfx_v11_0_rlc_backdoor_autoload_copy_ucode(struct amdgpu_device *adev,
1050                                               SOC21_FIRMWARE_ID id,
1051                                               const void *fw_data,
1052                                               uint32_t fw_size,
1053                                               uint32_t *fw_autoload_mask)
1054 {
1055         uint32_t toc_offset;
1056         uint32_t toc_fw_size;
1057         char *ptr = adev->gfx.rlc.rlc_autoload_ptr;
1058
1059         if (id <= SOC21_FIRMWARE_ID_INVALID || id >= SOC21_FIRMWARE_ID_MAX)
1060                 return;
1061
1062         toc_offset = rlc_autoload_info[id].offset;
1063         toc_fw_size = rlc_autoload_info[id].size;
1064
1065         if (fw_size == 0)
1066                 fw_size = toc_fw_size;
1067
1068         if (fw_size > toc_fw_size)
1069                 fw_size = toc_fw_size;
1070
1071         memcpy(ptr + toc_offset, fw_data, fw_size);
1072
1073         if (fw_size < toc_fw_size)
1074                 memset(ptr + toc_offset + fw_size, 0, toc_fw_size - fw_size);
1075
1076         if ((id != SOC21_FIRMWARE_ID_RS64_PFP) && (id != SOC21_FIRMWARE_ID_RS64_ME))
1077                 *(uint64_t *)fw_autoload_mask |= 1ULL << id;
1078 }
1079
1080 static void gfx_v11_0_rlc_backdoor_autoload_copy_toc_ucode(struct amdgpu_device *adev,
1081                                                         uint32_t *fw_autoload_mask)
1082 {
1083         void *data;
1084         uint32_t size;
1085         uint64_t *toc_ptr;
1086
1087         *(uint64_t *)fw_autoload_mask |= 0x1;
1088
1089         DRM_DEBUG("rlc autoload enabled fw: 0x%llx\n", *(uint64_t *)fw_autoload_mask);
1090
1091         data = adev->psp.toc.start_addr;
1092         size = rlc_autoload_info[SOC21_FIRMWARE_ID_RLC_TOC].size;
1093
1094         toc_ptr = (uint64_t *)data + size / 8 - 1;
1095         *toc_ptr = *(uint64_t *)fw_autoload_mask;
1096
1097         gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RLC_TOC,
1098                                         data, size, fw_autoload_mask);
1099 }
1100
1101 static void gfx_v11_0_rlc_backdoor_autoload_copy_gfx_ucode(struct amdgpu_device *adev,
1102                                                         uint32_t *fw_autoload_mask)
1103 {
1104         const __le32 *fw_data;
1105         uint32_t fw_size;
1106         const struct gfx_firmware_header_v1_0 *cp_hdr;
1107         const struct gfx_firmware_header_v2_0 *cpv2_hdr;
1108         const struct rlc_firmware_header_v2_0 *rlc_hdr;
1109         const struct rlc_firmware_header_v2_2 *rlcv22_hdr;
1110         uint16_t version_major, version_minor;
1111
1112         if (adev->gfx.rs64_enable) {
1113                 /* pfp ucode */
1114                 cpv2_hdr = (const struct gfx_firmware_header_v2_0 *)
1115                         adev->gfx.pfp_fw->data;
1116                 /* instruction */
1117                 fw_data = (const __le32 *)(adev->gfx.pfp_fw->data +
1118                         le32_to_cpu(cpv2_hdr->ucode_offset_bytes));
1119                 fw_size = le32_to_cpu(cpv2_hdr->ucode_size_bytes);
1120                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_PFP,
1121                                                 fw_data, fw_size, fw_autoload_mask);
1122                 /* data */
1123                 fw_data = (const __le32 *)(adev->gfx.pfp_fw->data +
1124                         le32_to_cpu(cpv2_hdr->data_offset_bytes));
1125                 fw_size = le32_to_cpu(cpv2_hdr->data_size_bytes);
1126                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_PFP_P0_STACK,
1127                                                 fw_data, fw_size, fw_autoload_mask);
1128                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_PFP_P1_STACK,
1129                                                 fw_data, fw_size, fw_autoload_mask);
1130                 /* me ucode */
1131                 cpv2_hdr = (const struct gfx_firmware_header_v2_0 *)
1132                         adev->gfx.me_fw->data;
1133                 /* instruction */
1134                 fw_data = (const __le32 *)(adev->gfx.me_fw->data +
1135                         le32_to_cpu(cpv2_hdr->ucode_offset_bytes));
1136                 fw_size = le32_to_cpu(cpv2_hdr->ucode_size_bytes);
1137                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_ME,
1138                                                 fw_data, fw_size, fw_autoload_mask);
1139                 /* data */
1140                 fw_data = (const __le32 *)(adev->gfx.me_fw->data +
1141                         le32_to_cpu(cpv2_hdr->data_offset_bytes));
1142                 fw_size = le32_to_cpu(cpv2_hdr->data_size_bytes);
1143                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_ME_P0_STACK,
1144                                                 fw_data, fw_size, fw_autoload_mask);
1145                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_ME_P1_STACK,
1146                                                 fw_data, fw_size, fw_autoload_mask);
1147                 /* mec ucode */
1148                 cpv2_hdr = (const struct gfx_firmware_header_v2_0 *)
1149                         adev->gfx.mec_fw->data;
1150                 /* instruction */
1151                 fw_data = (const __le32 *) (adev->gfx.mec_fw->data +
1152                         le32_to_cpu(cpv2_hdr->ucode_offset_bytes));
1153                 fw_size = le32_to_cpu(cpv2_hdr->ucode_size_bytes);
1154                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_MEC,
1155                                                 fw_data, fw_size, fw_autoload_mask);
1156                 /* data */
1157                 fw_data = (const __le32 *) (adev->gfx.mec_fw->data +
1158                         le32_to_cpu(cpv2_hdr->data_offset_bytes));
1159                 fw_size = le32_to_cpu(cpv2_hdr->data_size_bytes);
1160                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_MEC_P0_STACK,
1161                                                 fw_data, fw_size, fw_autoload_mask);
1162                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_MEC_P1_STACK,
1163                                                 fw_data, fw_size, fw_autoload_mask);
1164                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_MEC_P2_STACK,
1165                                                 fw_data, fw_size, fw_autoload_mask);
1166                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_MEC_P3_STACK,
1167                                                 fw_data, fw_size, fw_autoload_mask);
1168         } else {
1169                 /* pfp ucode */
1170                 cp_hdr = (const struct gfx_firmware_header_v1_0 *)
1171                         adev->gfx.pfp_fw->data;
1172                 fw_data = (const __le32 *)(adev->gfx.pfp_fw->data +
1173                                 le32_to_cpu(cp_hdr->header.ucode_array_offset_bytes));
1174                 fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes);
1175                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_CP_PFP,
1176                                                 fw_data, fw_size, fw_autoload_mask);
1177
1178                 /* me ucode */
1179                 cp_hdr = (const struct gfx_firmware_header_v1_0 *)
1180                         adev->gfx.me_fw->data;
1181                 fw_data = (const __le32 *)(adev->gfx.me_fw->data +
1182                                 le32_to_cpu(cp_hdr->header.ucode_array_offset_bytes));
1183                 fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes);
1184                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_CP_ME,
1185                                                 fw_data, fw_size, fw_autoload_mask);
1186
1187                 /* mec ucode */
1188                 cp_hdr = (const struct gfx_firmware_header_v1_0 *)
1189                         adev->gfx.mec_fw->data;
1190                 fw_data = (const __le32 *) (adev->gfx.mec_fw->data +
1191                                 le32_to_cpu(cp_hdr->header.ucode_array_offset_bytes));
1192                 fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes) -
1193                         cp_hdr->jt_size * 4;
1194                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_CP_MEC,
1195                                                 fw_data, fw_size, fw_autoload_mask);
1196         }
1197
1198         /* rlc ucode */
1199         rlc_hdr = (const struct rlc_firmware_header_v2_0 *)
1200                 adev->gfx.rlc_fw->data;
1201         fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1202                         le32_to_cpu(rlc_hdr->header.ucode_array_offset_bytes));
1203         fw_size = le32_to_cpu(rlc_hdr->header.ucode_size_bytes);
1204         gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RLC_G_UCODE,
1205                                         fw_data, fw_size, fw_autoload_mask);
1206
1207         version_major = le16_to_cpu(rlc_hdr->header.header_version_major);
1208         version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor);
1209         if (version_major == 2) {
1210                 if (version_minor >= 2) {
1211                         rlcv22_hdr = (const struct rlc_firmware_header_v2_2 *)adev->gfx.rlc_fw->data;
1212
1213                         fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1214                                         le32_to_cpu(rlcv22_hdr->rlc_iram_ucode_offset_bytes));
1215                         fw_size = le32_to_cpu(rlcv22_hdr->rlc_iram_ucode_size_bytes);
1216                         gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RLX6_UCODE,
1217                                         fw_data, fw_size, fw_autoload_mask);
1218
1219                         fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1220                                         le32_to_cpu(rlcv22_hdr->rlc_dram_ucode_offset_bytes));
1221                         fw_size = le32_to_cpu(rlcv22_hdr->rlc_dram_ucode_size_bytes);
1222                         gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RLX6_DRAM_BOOT,
1223                                         fw_data, fw_size, fw_autoload_mask);
1224                 }
1225         }
1226 }
1227
1228 static void gfx_v11_0_rlc_backdoor_autoload_copy_sdma_ucode(struct amdgpu_device *adev,
1229                                                         uint32_t *fw_autoload_mask)
1230 {
1231         const __le32 *fw_data;
1232         uint32_t fw_size;
1233         const struct sdma_firmware_header_v2_0 *sdma_hdr;
1234
1235         sdma_hdr = (const struct sdma_firmware_header_v2_0 *)
1236                 adev->sdma.instance[0].fw->data;
1237         fw_data = (const __le32 *) (adev->sdma.instance[0].fw->data +
1238                         le32_to_cpu(sdma_hdr->header.ucode_array_offset_bytes));
1239         fw_size = le32_to_cpu(sdma_hdr->ctx_ucode_size_bytes);
1240
1241         gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev,
1242                         SOC21_FIRMWARE_ID_SDMA_UCODE_TH0, fw_data, fw_size, fw_autoload_mask);
1243
1244         fw_data = (const __le32 *) (adev->sdma.instance[0].fw->data +
1245                         le32_to_cpu(sdma_hdr->ctl_ucode_offset));
1246         fw_size = le32_to_cpu(sdma_hdr->ctl_ucode_size_bytes);
1247
1248         gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev,
1249                         SOC21_FIRMWARE_ID_SDMA_UCODE_TH1, fw_data, fw_size, fw_autoload_mask);
1250 }
1251
1252 static void gfx_v11_0_rlc_backdoor_autoload_copy_mes_ucode(struct amdgpu_device *adev,
1253                                                         uint32_t *fw_autoload_mask)
1254 {
1255         const __le32 *fw_data;
1256         unsigned fw_size;
1257         const struct mes_firmware_header_v1_0 *mes_hdr;
1258         int pipe, ucode_id, data_id;
1259
1260         for (pipe = 0; pipe < 2; pipe++) {
1261                 if (pipe==0) {
1262                         ucode_id = SOC21_FIRMWARE_ID_RS64_MES_P0;
1263                         data_id  = SOC21_FIRMWARE_ID_RS64_MES_P0_STACK;
1264                 } else {
1265                         ucode_id = SOC21_FIRMWARE_ID_RS64_MES_P1;
1266                         data_id  = SOC21_FIRMWARE_ID_RS64_MES_P1_STACK;
1267                 }
1268
1269                 mes_hdr = (const struct mes_firmware_header_v1_0 *)
1270                         adev->mes.fw[pipe]->data;
1271
1272                 fw_data = (const __le32 *)(adev->mes.fw[pipe]->data +
1273                                 le32_to_cpu(mes_hdr->mes_ucode_offset_bytes));
1274                 fw_size = le32_to_cpu(mes_hdr->mes_ucode_size_bytes);
1275
1276                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev,
1277                                 ucode_id, fw_data, fw_size, fw_autoload_mask);
1278
1279                 fw_data = (const __le32 *)(adev->mes.fw[pipe]->data +
1280                                 le32_to_cpu(mes_hdr->mes_ucode_data_offset_bytes));
1281                 fw_size = le32_to_cpu(mes_hdr->mes_ucode_data_size_bytes);
1282
1283                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev,
1284                                 data_id, fw_data, fw_size, fw_autoload_mask);
1285         }
1286 }
1287
1288 static int gfx_v11_0_rlc_backdoor_autoload_enable(struct amdgpu_device *adev)
1289 {
1290         uint32_t rlc_g_offset, rlc_g_size;
1291         uint64_t gpu_addr;
1292         uint32_t autoload_fw_id[2];
1293
1294         memset(autoload_fw_id, 0, sizeof(uint32_t) * 2);
1295
1296         /* RLC autoload sequence 2: copy ucode */
1297         gfx_v11_0_rlc_backdoor_autoload_copy_sdma_ucode(adev, autoload_fw_id);
1298         gfx_v11_0_rlc_backdoor_autoload_copy_gfx_ucode(adev, autoload_fw_id);
1299         gfx_v11_0_rlc_backdoor_autoload_copy_mes_ucode(adev, autoload_fw_id);
1300         gfx_v11_0_rlc_backdoor_autoload_copy_toc_ucode(adev, autoload_fw_id);
1301
1302         rlc_g_offset = rlc_autoload_info[SOC21_FIRMWARE_ID_RLC_G_UCODE].offset;
1303         rlc_g_size = rlc_autoload_info[SOC21_FIRMWARE_ID_RLC_G_UCODE].size;
1304         gpu_addr = adev->gfx.rlc.rlc_autoload_gpu_addr + rlc_g_offset;
1305
1306         WREG32_SOC15(GC, 0, regGFX_IMU_RLC_BOOTLOADER_ADDR_HI, upper_32_bits(gpu_addr));
1307         WREG32_SOC15(GC, 0, regGFX_IMU_RLC_BOOTLOADER_ADDR_LO, lower_32_bits(gpu_addr));
1308
1309         WREG32_SOC15(GC, 0, regGFX_IMU_RLC_BOOTLOADER_SIZE, rlc_g_size);
1310
1311         /* RLC autoload sequence 3: load IMU fw */
1312         if (adev->gfx.imu.funcs->load_microcode)
1313                 adev->gfx.imu.funcs->load_microcode(adev);
1314         /* RLC autoload sequence 4 init IMU fw */
1315         if (adev->gfx.imu.funcs->setup_imu)
1316                 adev->gfx.imu.funcs->setup_imu(adev);
1317         if (adev->gfx.imu.funcs->start_imu)
1318                 adev->gfx.imu.funcs->start_imu(adev);
1319
1320         /* RLC autoload sequence 5 disable gpa mode */
1321         gfx_v11_0_disable_gpa_mode(adev);
1322
1323         return 0;
1324 }
1325
1326 static int gfx_v11_0_sw_init(void *handle)
1327 {
1328         int i, j, k, r, ring_id = 0;
1329         struct amdgpu_kiq *kiq;
1330         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1331
1332         switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
1333         case IP_VERSION(11, 0, 0):
1334         case IP_VERSION(11, 0, 2):
1335         case IP_VERSION(11, 0, 3):
1336                 adev->gfx.me.num_me = 1;
1337                 adev->gfx.me.num_pipe_per_me = 1;
1338                 adev->gfx.me.num_queue_per_pipe = 1;
1339                 adev->gfx.mec.num_mec = 2;
1340                 adev->gfx.mec.num_pipe_per_mec = 4;
1341                 adev->gfx.mec.num_queue_per_pipe = 4;
1342                 break;
1343         case IP_VERSION(11, 0, 1):
1344         case IP_VERSION(11, 0, 4):
1345         case IP_VERSION(11, 5, 0):
1346                 adev->gfx.me.num_me = 1;
1347                 adev->gfx.me.num_pipe_per_me = 1;
1348                 adev->gfx.me.num_queue_per_pipe = 1;
1349                 adev->gfx.mec.num_mec = 1;
1350                 adev->gfx.mec.num_pipe_per_mec = 4;
1351                 adev->gfx.mec.num_queue_per_pipe = 4;
1352                 break;
1353         default:
1354                 adev->gfx.me.num_me = 1;
1355                 adev->gfx.me.num_pipe_per_me = 1;
1356                 adev->gfx.me.num_queue_per_pipe = 1;
1357                 adev->gfx.mec.num_mec = 1;
1358                 adev->gfx.mec.num_pipe_per_mec = 4;
1359                 adev->gfx.mec.num_queue_per_pipe = 8;
1360                 break;
1361         }
1362
1363         /* Enable CG flag in one VF mode for enabling RLC safe mode enter/exit */
1364         if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 0, 3) &&
1365             amdgpu_sriov_is_pp_one_vf(adev))
1366                 adev->cg_flags = AMD_CG_SUPPORT_GFX_CGCG;
1367
1368         /* EOP Event */
1369         r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GRBM_CP,
1370                               GFX_11_0_0__SRCID__CP_EOP_INTERRUPT,
1371                               &adev->gfx.eop_irq);
1372         if (r)
1373                 return r;
1374
1375         /* Privileged reg */
1376         r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GRBM_CP,
1377                               GFX_11_0_0__SRCID__CP_PRIV_REG_FAULT,
1378                               &adev->gfx.priv_reg_irq);
1379         if (r)
1380                 return r;
1381
1382         /* Privileged inst */
1383         r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GRBM_CP,
1384                               GFX_11_0_0__SRCID__CP_PRIV_INSTR_FAULT,
1385                               &adev->gfx.priv_inst_irq);
1386         if (r)
1387                 return r;
1388
1389         /* FED error */
1390         r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GFX,
1391                                   GFX_11_0_0__SRCID__RLC_GC_FED_INTERRUPT,
1392                                   &adev->gfx.rlc_gc_fed_irq);
1393         if (r)
1394                 return r;
1395
1396         adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE;
1397
1398         if (adev->gfx.imu.funcs) {
1399                 if (adev->gfx.imu.funcs->init_microcode) {
1400                         r = adev->gfx.imu.funcs->init_microcode(adev);
1401                         if (r)
1402                                 DRM_ERROR("Failed to load imu firmware!\n");
1403                 }
1404         }
1405
1406         gfx_v11_0_me_init(adev);
1407
1408         r = gfx_v11_0_rlc_init(adev);
1409         if (r) {
1410                 DRM_ERROR("Failed to init rlc BOs!\n");
1411                 return r;
1412         }
1413
1414         r = gfx_v11_0_mec_init(adev);
1415         if (r) {
1416                 DRM_ERROR("Failed to init MEC BOs!\n");
1417                 return r;
1418         }
1419
1420         /* set up the gfx ring */
1421         for (i = 0; i < adev->gfx.me.num_me; i++) {
1422                 for (j = 0; j < adev->gfx.me.num_queue_per_pipe; j++) {
1423                         for (k = 0; k < adev->gfx.me.num_pipe_per_me; k++) {
1424                                 if (!amdgpu_gfx_is_me_queue_enabled(adev, i, k, j))
1425                                         continue;
1426
1427                                 r = gfx_v11_0_gfx_ring_init(adev, ring_id,
1428                                                             i, k, j);
1429                                 if (r)
1430                                         return r;
1431                                 ring_id++;
1432                         }
1433                 }
1434         }
1435
1436         ring_id = 0;
1437         /* set up the compute queues - allocate horizontally across pipes */
1438         for (i = 0; i < adev->gfx.mec.num_mec; ++i) {
1439                 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {
1440                         for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {
1441                                 if (!amdgpu_gfx_is_mec_queue_enabled(adev, 0, i,
1442                                                                      k, j))
1443                                         continue;
1444
1445                                 r = gfx_v11_0_compute_ring_init(adev, ring_id,
1446                                                                 i, k, j);
1447                                 if (r)
1448                                         return r;
1449
1450                                 ring_id++;
1451                         }
1452                 }
1453         }
1454
1455         if (!adev->enable_mes_kiq) {
1456                 r = amdgpu_gfx_kiq_init(adev, GFX11_MEC_HPD_SIZE, 0);
1457                 if (r) {
1458                         DRM_ERROR("Failed to init KIQ BOs!\n");
1459                         return r;
1460                 }
1461
1462                 kiq = &adev->gfx.kiq[0];
1463                 r = amdgpu_gfx_kiq_init_ring(adev, &kiq->ring, &kiq->irq, 0);
1464                 if (r)
1465                         return r;
1466         }
1467
1468         r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct v11_compute_mqd), 0);
1469         if (r)
1470                 return r;
1471
1472         /* allocate visible FB for rlc auto-loading fw */
1473         if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) {
1474                 r = gfx_v11_0_rlc_autoload_buffer_init(adev);
1475                 if (r)
1476                         return r;
1477         }
1478
1479         r = gfx_v11_0_gpu_early_init(adev);
1480         if (r)
1481                 return r;
1482
1483         if (amdgpu_gfx_ras_sw_init(adev)) {
1484                 dev_err(adev->dev, "Failed to initialize gfx ras block!\n");
1485                 return -EINVAL;
1486         }
1487
1488         return 0;
1489 }
1490
1491 static void gfx_v11_0_pfp_fini(struct amdgpu_device *adev)
1492 {
1493         amdgpu_bo_free_kernel(&adev->gfx.pfp.pfp_fw_obj,
1494                               &adev->gfx.pfp.pfp_fw_gpu_addr,
1495                               (void **)&adev->gfx.pfp.pfp_fw_ptr);
1496
1497         amdgpu_bo_free_kernel(&adev->gfx.pfp.pfp_fw_data_obj,
1498                               &adev->gfx.pfp.pfp_fw_data_gpu_addr,
1499                               (void **)&adev->gfx.pfp.pfp_fw_data_ptr);
1500 }
1501
1502 static void gfx_v11_0_me_fini(struct amdgpu_device *adev)
1503 {
1504         amdgpu_bo_free_kernel(&adev->gfx.me.me_fw_obj,
1505                               &adev->gfx.me.me_fw_gpu_addr,
1506                               (void **)&adev->gfx.me.me_fw_ptr);
1507
1508         amdgpu_bo_free_kernel(&adev->gfx.me.me_fw_data_obj,
1509                                &adev->gfx.me.me_fw_data_gpu_addr,
1510                                (void **)&adev->gfx.me.me_fw_data_ptr);
1511 }
1512
1513 static void gfx_v11_0_rlc_autoload_buffer_fini(struct amdgpu_device *adev)
1514 {
1515         amdgpu_bo_free_kernel(&adev->gfx.rlc.rlc_autoload_bo,
1516                         &adev->gfx.rlc.rlc_autoload_gpu_addr,
1517                         (void **)&adev->gfx.rlc.rlc_autoload_ptr);
1518 }
1519
1520 static int gfx_v11_0_sw_fini(void *handle)
1521 {
1522         int i;
1523         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1524
1525         for (i = 0; i < adev->gfx.num_gfx_rings; i++)
1526                 amdgpu_ring_fini(&adev->gfx.gfx_ring[i]);
1527         for (i = 0; i < adev->gfx.num_compute_rings; i++)
1528                 amdgpu_ring_fini(&adev->gfx.compute_ring[i]);
1529
1530         amdgpu_gfx_mqd_sw_fini(adev, 0);
1531
1532         if (!adev->enable_mes_kiq) {
1533                 amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq[0].ring);
1534                 amdgpu_gfx_kiq_fini(adev, 0);
1535         }
1536
1537         gfx_v11_0_pfp_fini(adev);
1538         gfx_v11_0_me_fini(adev);
1539         gfx_v11_0_rlc_fini(adev);
1540         gfx_v11_0_mec_fini(adev);
1541
1542         if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO)
1543                 gfx_v11_0_rlc_autoload_buffer_fini(adev);
1544
1545         gfx_v11_0_free_microcode(adev);
1546
1547         return 0;
1548 }
1549
1550 static void gfx_v11_0_select_se_sh(struct amdgpu_device *adev, u32 se_num,
1551                                    u32 sh_num, u32 instance, int xcc_id)
1552 {
1553         u32 data;
1554
1555         if (instance == 0xffffffff)
1556                 data = REG_SET_FIELD(0, GRBM_GFX_INDEX,
1557                                      INSTANCE_BROADCAST_WRITES, 1);
1558         else
1559                 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX,
1560                                      instance);
1561
1562         if (se_num == 0xffffffff)
1563                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES,
1564                                      1);
1565         else
1566                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num);
1567
1568         if (sh_num == 0xffffffff)
1569                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SA_BROADCAST_WRITES,
1570                                      1);
1571         else
1572                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SA_INDEX, sh_num);
1573
1574         WREG32_SOC15(GC, 0, regGRBM_GFX_INDEX, data);
1575 }
1576
1577 static u32 gfx_v11_0_get_sa_active_bitmap(struct amdgpu_device *adev)
1578 {
1579         u32 gc_disabled_sa_mask, gc_user_disabled_sa_mask, sa_mask;
1580
1581         gc_disabled_sa_mask = RREG32_SOC15(GC, 0, regCC_GC_SA_UNIT_DISABLE);
1582         gc_disabled_sa_mask = REG_GET_FIELD(gc_disabled_sa_mask,
1583                                            CC_GC_SA_UNIT_DISABLE,
1584                                            SA_DISABLE);
1585         gc_user_disabled_sa_mask = RREG32_SOC15(GC, 0, regGC_USER_SA_UNIT_DISABLE);
1586         gc_user_disabled_sa_mask = REG_GET_FIELD(gc_user_disabled_sa_mask,
1587                                                  GC_USER_SA_UNIT_DISABLE,
1588                                                  SA_DISABLE);
1589         sa_mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_sh_per_se *
1590                                             adev->gfx.config.max_shader_engines);
1591
1592         return sa_mask & (~(gc_disabled_sa_mask | gc_user_disabled_sa_mask));
1593 }
1594
1595 static u32 gfx_v11_0_get_rb_active_bitmap(struct amdgpu_device *adev)
1596 {
1597         u32 gc_disabled_rb_mask, gc_user_disabled_rb_mask;
1598         u32 rb_mask;
1599
1600         gc_disabled_rb_mask = RREG32_SOC15(GC, 0, regCC_RB_BACKEND_DISABLE);
1601         gc_disabled_rb_mask = REG_GET_FIELD(gc_disabled_rb_mask,
1602                                             CC_RB_BACKEND_DISABLE,
1603                                             BACKEND_DISABLE);
1604         gc_user_disabled_rb_mask = RREG32_SOC15(GC, 0, regGC_USER_RB_BACKEND_DISABLE);
1605         gc_user_disabled_rb_mask = REG_GET_FIELD(gc_user_disabled_rb_mask,
1606                                                  GC_USER_RB_BACKEND_DISABLE,
1607                                                  BACKEND_DISABLE);
1608         rb_mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se *
1609                                             adev->gfx.config.max_shader_engines);
1610
1611         return rb_mask & (~(gc_disabled_rb_mask | gc_user_disabled_rb_mask));
1612 }
1613
1614 static void gfx_v11_0_setup_rb(struct amdgpu_device *adev)
1615 {
1616         u32 rb_bitmap_width_per_sa;
1617         u32 max_sa;
1618         u32 active_sa_bitmap;
1619         u32 global_active_rb_bitmap;
1620         u32 active_rb_bitmap = 0;
1621         u32 i;
1622
1623         /* query sa bitmap from SA_UNIT_DISABLE registers */
1624         active_sa_bitmap = gfx_v11_0_get_sa_active_bitmap(adev);
1625         /* query rb bitmap from RB_BACKEND_DISABLE registers */
1626         global_active_rb_bitmap = gfx_v11_0_get_rb_active_bitmap(adev);
1627
1628         /* generate active rb bitmap according to active sa bitmap */
1629         max_sa = adev->gfx.config.max_shader_engines *
1630                  adev->gfx.config.max_sh_per_se;
1631         rb_bitmap_width_per_sa = adev->gfx.config.max_backends_per_se /
1632                                  adev->gfx.config.max_sh_per_se;
1633         for (i = 0; i < max_sa; i++) {
1634                 if (active_sa_bitmap & (1 << i))
1635                         active_rb_bitmap |= (0x3 << (i * rb_bitmap_width_per_sa));
1636         }
1637
1638         active_rb_bitmap |= global_active_rb_bitmap;
1639         adev->gfx.config.backend_enable_mask = active_rb_bitmap;
1640         adev->gfx.config.num_rbs = hweight32(active_rb_bitmap);
1641 }
1642
1643 #define DEFAULT_SH_MEM_BASES    (0x6000)
1644 #define LDS_APP_BASE           0x1
1645 #define SCRATCH_APP_BASE       0x2
1646
1647 static void gfx_v11_0_init_compute_vmid(struct amdgpu_device *adev)
1648 {
1649         int i;
1650         uint32_t sh_mem_bases;
1651         uint32_t data;
1652
1653         /*
1654          * Configure apertures:
1655          * LDS:         0x60000000'00000000 - 0x60000001'00000000 (4GB)
1656          * Scratch:     0x60000001'00000000 - 0x60000002'00000000 (4GB)
1657          * GPUVM:       0x60010000'00000000 - 0x60020000'00000000 (1TB)
1658          */
1659         sh_mem_bases = (LDS_APP_BASE << SH_MEM_BASES__SHARED_BASE__SHIFT) |
1660                         SCRATCH_APP_BASE;
1661
1662         mutex_lock(&adev->srbm_mutex);
1663         for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
1664                 soc21_grbm_select(adev, 0, 0, 0, i);
1665                 /* CP and shaders */
1666                 WREG32_SOC15(GC, 0, regSH_MEM_CONFIG, DEFAULT_SH_MEM_CONFIG);
1667                 WREG32_SOC15(GC, 0, regSH_MEM_BASES, sh_mem_bases);
1668
1669                 /* Enable trap for each kfd vmid. */
1670                 data = RREG32_SOC15(GC, 0, regSPI_GDBG_PER_VMID_CNTL);
1671                 data = REG_SET_FIELD(data, SPI_GDBG_PER_VMID_CNTL, TRAP_EN, 1);
1672                 WREG32_SOC15(GC, 0, regSPI_GDBG_PER_VMID_CNTL, data);
1673         }
1674         soc21_grbm_select(adev, 0, 0, 0, 0);
1675         mutex_unlock(&adev->srbm_mutex);
1676
1677         /* Initialize all compute VMIDs to have no GDS, GWS, or OA
1678            acccess. These should be enabled by FW for target VMIDs. */
1679         for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
1680                 WREG32_SOC15_OFFSET(GC, 0, regGDS_VMID0_BASE, 2 * i, 0);
1681                 WREG32_SOC15_OFFSET(GC, 0, regGDS_VMID0_SIZE, 2 * i, 0);
1682                 WREG32_SOC15_OFFSET(GC, 0, regGDS_GWS_VMID0, i, 0);
1683                 WREG32_SOC15_OFFSET(GC, 0, regGDS_OA_VMID0, i, 0);
1684         }
1685 }
1686
1687 static void gfx_v11_0_init_gds_vmid(struct amdgpu_device *adev)
1688 {
1689         int vmid;
1690
1691         /*
1692          * Initialize all compute and user-gfx VMIDs to have no GDS, GWS, or OA
1693          * access. Compute VMIDs should be enabled by FW for target VMIDs,
1694          * the driver can enable them for graphics. VMID0 should maintain
1695          * access so that HWS firmware can save/restore entries.
1696          */
1697         for (vmid = 1; vmid < 16; vmid++) {
1698                 WREG32_SOC15_OFFSET(GC, 0, regGDS_VMID0_BASE, 2 * vmid, 0);
1699                 WREG32_SOC15_OFFSET(GC, 0, regGDS_VMID0_SIZE, 2 * vmid, 0);
1700                 WREG32_SOC15_OFFSET(GC, 0, regGDS_GWS_VMID0, vmid, 0);
1701                 WREG32_SOC15_OFFSET(GC, 0, regGDS_OA_VMID0, vmid, 0);
1702         }
1703 }
1704
1705 static void gfx_v11_0_tcp_harvest(struct amdgpu_device *adev)
1706 {
1707         /* TODO: harvest feature to be added later. */
1708 }
1709
1710 static void gfx_v11_0_get_tcc_info(struct amdgpu_device *adev)
1711 {
1712         /* TCCs are global (not instanced). */
1713         uint32_t tcc_disable = RREG32_SOC15(GC, 0, regCGTS_TCC_DISABLE) |
1714                                RREG32_SOC15(GC, 0, regCGTS_USER_TCC_DISABLE);
1715
1716         adev->gfx.config.tcc_disabled_mask =
1717                 REG_GET_FIELD(tcc_disable, CGTS_TCC_DISABLE, TCC_DISABLE) |
1718                 (REG_GET_FIELD(tcc_disable, CGTS_TCC_DISABLE, HI_TCC_DISABLE) << 16);
1719 }
1720
1721 static void gfx_v11_0_constants_init(struct amdgpu_device *adev)
1722 {
1723         u32 tmp;
1724         int i;
1725
1726         if (!amdgpu_sriov_vf(adev))
1727                 WREG32_FIELD15_PREREG(GC, 0, GRBM_CNTL, READ_TIMEOUT, 0xff);
1728
1729         gfx_v11_0_setup_rb(adev);
1730         gfx_v11_0_get_cu_info(adev, &adev->gfx.cu_info);
1731         gfx_v11_0_get_tcc_info(adev);
1732         adev->gfx.config.pa_sc_tile_steering_override = 0;
1733
1734         /* Set whether texture coordinate truncation is conformant. */
1735         tmp = RREG32_SOC15(GC, 0, regTA_CNTL2);
1736         adev->gfx.config.ta_cntl2_truncate_coord_mode =
1737                 REG_GET_FIELD(tmp, TA_CNTL2, TRUNCATE_COORD_MODE);
1738
1739         /* XXX SH_MEM regs */
1740         /* where to put LDS, scratch, GPUVM in FSA64 space */
1741         mutex_lock(&adev->srbm_mutex);
1742         for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB(0)].num_ids; i++) {
1743                 soc21_grbm_select(adev, 0, 0, 0, i);
1744                 /* CP and shaders */
1745                 WREG32_SOC15(GC, 0, regSH_MEM_CONFIG, DEFAULT_SH_MEM_CONFIG);
1746                 if (i != 0) {
1747                         tmp = REG_SET_FIELD(0, SH_MEM_BASES, PRIVATE_BASE,
1748                                 (adev->gmc.private_aperture_start >> 48));
1749                         tmp = REG_SET_FIELD(tmp, SH_MEM_BASES, SHARED_BASE,
1750                                 (adev->gmc.shared_aperture_start >> 48));
1751                         WREG32_SOC15(GC, 0, regSH_MEM_BASES, tmp);
1752                 }
1753         }
1754         soc21_grbm_select(adev, 0, 0, 0, 0);
1755
1756         mutex_unlock(&adev->srbm_mutex);
1757
1758         gfx_v11_0_init_compute_vmid(adev);
1759         gfx_v11_0_init_gds_vmid(adev);
1760 }
1761
1762 static void gfx_v11_0_enable_gui_idle_interrupt(struct amdgpu_device *adev,
1763                                                bool enable)
1764 {
1765         u32 tmp;
1766
1767         if (amdgpu_sriov_vf(adev))
1768                 return;
1769
1770         tmp = RREG32_SOC15(GC, 0, regCP_INT_CNTL_RING0);
1771
1772         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE,
1773                             enable ? 1 : 0);
1774         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE,
1775                             enable ? 1 : 0);
1776         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE,
1777                             enable ? 1 : 0);
1778         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE,
1779                             enable ? 1 : 0);
1780
1781         WREG32_SOC15(GC, 0, regCP_INT_CNTL_RING0, tmp);
1782 }
1783
1784 static int gfx_v11_0_init_csb(struct amdgpu_device *adev)
1785 {
1786         adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr);
1787
1788         WREG32_SOC15(GC, 0, regRLC_CSIB_ADDR_HI,
1789                         adev->gfx.rlc.clear_state_gpu_addr >> 32);
1790         WREG32_SOC15(GC, 0, regRLC_CSIB_ADDR_LO,
1791                         adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc);
1792         WREG32_SOC15(GC, 0, regRLC_CSIB_LENGTH, adev->gfx.rlc.clear_state_size);
1793
1794         return 0;
1795 }
1796
1797 static void gfx_v11_0_rlc_stop(struct amdgpu_device *adev)
1798 {
1799         u32 tmp = RREG32_SOC15(GC, 0, regRLC_CNTL);
1800
1801         tmp = REG_SET_FIELD(tmp, RLC_CNTL, RLC_ENABLE_F32, 0);
1802         WREG32_SOC15(GC, 0, regRLC_CNTL, tmp);
1803 }
1804
1805 static void gfx_v11_0_rlc_reset(struct amdgpu_device *adev)
1806 {
1807         WREG32_FIELD15_PREREG(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
1808         udelay(50);
1809         WREG32_FIELD15_PREREG(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 0);
1810         udelay(50);
1811 }
1812
1813 static void gfx_v11_0_rlc_smu_handshake_cntl(struct amdgpu_device *adev,
1814                                              bool enable)
1815 {
1816         uint32_t rlc_pg_cntl;
1817
1818         rlc_pg_cntl = RREG32_SOC15(GC, 0, regRLC_PG_CNTL);
1819
1820         if (!enable) {
1821                 /* RLC_PG_CNTL[23] = 0 (default)
1822                  * RLC will wait for handshake acks with SMU
1823                  * GFXOFF will be enabled
1824                  * RLC_PG_CNTL[23] = 1
1825                  * RLC will not issue any message to SMU
1826                  * hence no handshake between SMU & RLC
1827                  * GFXOFF will be disabled
1828                  */
1829                 rlc_pg_cntl |= RLC_PG_CNTL__SMU_HANDSHAKE_DISABLE_MASK;
1830         } else
1831                 rlc_pg_cntl &= ~RLC_PG_CNTL__SMU_HANDSHAKE_DISABLE_MASK;
1832         WREG32_SOC15(GC, 0, regRLC_PG_CNTL, rlc_pg_cntl);
1833 }
1834
1835 static void gfx_v11_0_rlc_start(struct amdgpu_device *adev)
1836 {
1837         /* TODO: enable rlc & smu handshake until smu
1838          * and gfxoff feature works as expected */
1839         if (!(amdgpu_pp_feature_mask & PP_GFXOFF_MASK))
1840                 gfx_v11_0_rlc_smu_handshake_cntl(adev, false);
1841
1842         WREG32_FIELD15_PREREG(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 1);
1843         udelay(50);
1844 }
1845
1846 static void gfx_v11_0_rlc_enable_srm(struct amdgpu_device *adev)
1847 {
1848         uint32_t tmp;
1849
1850         /* enable Save Restore Machine */
1851         tmp = RREG32(SOC15_REG_OFFSET(GC, 0, regRLC_SRM_CNTL));
1852         tmp |= RLC_SRM_CNTL__AUTO_INCR_ADDR_MASK;
1853         tmp |= RLC_SRM_CNTL__SRM_ENABLE_MASK;
1854         WREG32(SOC15_REG_OFFSET(GC, 0, regRLC_SRM_CNTL), tmp);
1855 }
1856
1857 static void gfx_v11_0_load_rlcg_microcode(struct amdgpu_device *adev)
1858 {
1859         const struct rlc_firmware_header_v2_0 *hdr;
1860         const __le32 *fw_data;
1861         unsigned i, fw_size;
1862
1863         hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
1864         fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1865                            le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1866         fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
1867
1868         WREG32_SOC15(GC, 0, regRLC_GPM_UCODE_ADDR,
1869                      RLCG_UCODE_LOADING_START_ADDRESS);
1870
1871         for (i = 0; i < fw_size; i++)
1872                 WREG32_SOC15(GC, 0, regRLC_GPM_UCODE_DATA,
1873                              le32_to_cpup(fw_data++));
1874
1875         WREG32_SOC15(GC, 0, regRLC_GPM_UCODE_ADDR, adev->gfx.rlc_fw_version);
1876 }
1877
1878 static void gfx_v11_0_load_rlc_iram_dram_microcode(struct amdgpu_device *adev)
1879 {
1880         const struct rlc_firmware_header_v2_2 *hdr;
1881         const __le32 *fw_data;
1882         unsigned i, fw_size;
1883         u32 tmp;
1884
1885         hdr = (const struct rlc_firmware_header_v2_2 *)adev->gfx.rlc_fw->data;
1886
1887         fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1888                         le32_to_cpu(hdr->rlc_iram_ucode_offset_bytes));
1889         fw_size = le32_to_cpu(hdr->rlc_iram_ucode_size_bytes) / 4;
1890
1891         WREG32_SOC15(GC, 0, regRLC_LX6_IRAM_ADDR, 0);
1892
1893         for (i = 0; i < fw_size; i++) {
1894                 if ((amdgpu_emu_mode == 1) && (i % 100 == 99))
1895                         msleep(1);
1896                 WREG32_SOC15(GC, 0, regRLC_LX6_IRAM_DATA,
1897                                 le32_to_cpup(fw_data++));
1898         }
1899
1900         WREG32_SOC15(GC, 0, regRLC_LX6_IRAM_ADDR, adev->gfx.rlc_fw_version);
1901
1902         fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1903                         le32_to_cpu(hdr->rlc_dram_ucode_offset_bytes));
1904         fw_size = le32_to_cpu(hdr->rlc_dram_ucode_size_bytes) / 4;
1905
1906         WREG32_SOC15(GC, 0, regRLC_LX6_DRAM_ADDR, 0);
1907         for (i = 0; i < fw_size; i++) {
1908                 if ((amdgpu_emu_mode == 1) && (i % 100 == 99))
1909                         msleep(1);
1910                 WREG32_SOC15(GC, 0, regRLC_LX6_DRAM_DATA,
1911                                 le32_to_cpup(fw_data++));
1912         }
1913
1914         WREG32_SOC15(GC, 0, regRLC_LX6_IRAM_ADDR, adev->gfx.rlc_fw_version);
1915
1916         tmp = RREG32_SOC15(GC, 0, regRLC_LX6_CNTL);
1917         tmp = REG_SET_FIELD(tmp, RLC_LX6_CNTL, PDEBUG_ENABLE, 1);
1918         tmp = REG_SET_FIELD(tmp, RLC_LX6_CNTL, BRESET, 0);
1919         WREG32_SOC15(GC, 0, regRLC_LX6_CNTL, tmp);
1920 }
1921
1922 static void gfx_v11_0_load_rlcp_rlcv_microcode(struct amdgpu_device *adev)
1923 {
1924         const struct rlc_firmware_header_v2_3 *hdr;
1925         const __le32 *fw_data;
1926         unsigned i, fw_size;
1927         u32 tmp;
1928
1929         hdr = (const struct rlc_firmware_header_v2_3 *)adev->gfx.rlc_fw->data;
1930
1931         fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1932                         le32_to_cpu(hdr->rlcp_ucode_offset_bytes));
1933         fw_size = le32_to_cpu(hdr->rlcp_ucode_size_bytes) / 4;
1934
1935         WREG32_SOC15(GC, 0, regRLC_PACE_UCODE_ADDR, 0);
1936
1937         for (i = 0; i < fw_size; i++) {
1938                 if ((amdgpu_emu_mode == 1) && (i % 100 == 99))
1939                         msleep(1);
1940                 WREG32_SOC15(GC, 0, regRLC_PACE_UCODE_DATA,
1941                                 le32_to_cpup(fw_data++));
1942         }
1943
1944         WREG32_SOC15(GC, 0, regRLC_PACE_UCODE_ADDR, adev->gfx.rlc_fw_version);
1945
1946         tmp = RREG32_SOC15(GC, 0, regRLC_GPM_THREAD_ENABLE);
1947         tmp = REG_SET_FIELD(tmp, RLC_GPM_THREAD_ENABLE, THREAD1_ENABLE, 1);
1948         WREG32_SOC15(GC, 0, regRLC_GPM_THREAD_ENABLE, tmp);
1949
1950         fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1951                         le32_to_cpu(hdr->rlcv_ucode_offset_bytes));
1952         fw_size = le32_to_cpu(hdr->rlcv_ucode_size_bytes) / 4;
1953
1954         WREG32_SOC15(GC, 0, regRLC_GPU_IOV_UCODE_ADDR, 0);
1955
1956         for (i = 0; i < fw_size; i++) {
1957                 if ((amdgpu_emu_mode == 1) && (i % 100 == 99))
1958                         msleep(1);
1959                 WREG32_SOC15(GC, 0, regRLC_GPU_IOV_UCODE_DATA,
1960                                 le32_to_cpup(fw_data++));
1961         }
1962
1963         WREG32_SOC15(GC, 0, regRLC_GPU_IOV_UCODE_ADDR, adev->gfx.rlc_fw_version);
1964
1965         tmp = RREG32_SOC15(GC, 0, regRLC_GPU_IOV_F32_CNTL);
1966         tmp = REG_SET_FIELD(tmp, RLC_GPU_IOV_F32_CNTL, ENABLE, 1);
1967         WREG32_SOC15(GC, 0, regRLC_GPU_IOV_F32_CNTL, tmp);
1968 }
1969
1970 static int gfx_v11_0_rlc_load_microcode(struct amdgpu_device *adev)
1971 {
1972         const struct rlc_firmware_header_v2_0 *hdr;
1973         uint16_t version_major;
1974         uint16_t version_minor;
1975
1976         if (!adev->gfx.rlc_fw)
1977                 return -EINVAL;
1978
1979         hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
1980         amdgpu_ucode_print_rlc_hdr(&hdr->header);
1981
1982         version_major = le16_to_cpu(hdr->header.header_version_major);
1983         version_minor = le16_to_cpu(hdr->header.header_version_minor);
1984
1985         if (version_major == 2) {
1986                 gfx_v11_0_load_rlcg_microcode(adev);
1987                 if (amdgpu_dpm == 1) {
1988                         if (version_minor >= 2)
1989                                 gfx_v11_0_load_rlc_iram_dram_microcode(adev);
1990                         if (version_minor == 3)
1991                                 gfx_v11_0_load_rlcp_rlcv_microcode(adev);
1992                 }
1993
1994                 return 0;
1995         }
1996
1997         return -EINVAL;
1998 }
1999
2000 static int gfx_v11_0_rlc_resume(struct amdgpu_device *adev)
2001 {
2002         int r;
2003
2004         if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
2005                 gfx_v11_0_init_csb(adev);
2006
2007                 if (!amdgpu_sriov_vf(adev)) /* enable RLC SRM */
2008                         gfx_v11_0_rlc_enable_srm(adev);
2009         } else {
2010                 if (amdgpu_sriov_vf(adev)) {
2011                         gfx_v11_0_init_csb(adev);
2012                         return 0;
2013                 }
2014
2015                 adev->gfx.rlc.funcs->stop(adev);
2016
2017                 /* disable CG */
2018                 WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL, 0);
2019
2020                 /* disable PG */
2021                 WREG32_SOC15(GC, 0, regRLC_PG_CNTL, 0);
2022
2023                 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
2024                         /* legacy rlc firmware loading */
2025                         r = gfx_v11_0_rlc_load_microcode(adev);
2026                         if (r)
2027                                 return r;
2028                 }
2029
2030                 gfx_v11_0_init_csb(adev);
2031
2032                 adev->gfx.rlc.funcs->start(adev);
2033         }
2034         return 0;
2035 }
2036
2037 static int gfx_v11_0_config_me_cache(struct amdgpu_device *adev, uint64_t addr)
2038 {
2039         uint32_t usec_timeout = 50000;  /* wait for 50ms */
2040         uint32_t tmp;
2041         int i;
2042
2043         /* Trigger an invalidation of the L1 instruction caches */
2044         tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
2045         tmp = REG_SET_FIELD(tmp, CP_ME_IC_OP_CNTL, INVALIDATE_CACHE, 1);
2046         WREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL, tmp);
2047
2048         /* Wait for invalidation complete */
2049         for (i = 0; i < usec_timeout; i++) {
2050                 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
2051                 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL,
2052                                         INVALIDATE_CACHE_COMPLETE))
2053                         break;
2054                 udelay(1);
2055         }
2056
2057         if (i >= usec_timeout) {
2058                 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2059                 return -EINVAL;
2060         }
2061
2062         if (amdgpu_emu_mode == 1)
2063                 adev->hdp.funcs->flush_hdp(adev, NULL);
2064
2065         tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL);
2066         tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, VMID, 0);
2067         tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, CACHE_POLICY, 0);
2068         tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, EXE_DISABLE, 0);
2069         tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, ADDRESS_CLAMP, 1);
2070         WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL, tmp);
2071
2072         /* Program me ucode address into intruction cache address register */
2073         WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_LO,
2074                         lower_32_bits(addr) & 0xFFFFF000);
2075         WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_HI,
2076                         upper_32_bits(addr));
2077
2078         return 0;
2079 }
2080
2081 static int gfx_v11_0_config_pfp_cache(struct amdgpu_device *adev, uint64_t addr)
2082 {
2083         uint32_t usec_timeout = 50000;  /* wait for 50ms */
2084         uint32_t tmp;
2085         int i;
2086
2087         /* Trigger an invalidation of the L1 instruction caches */
2088         tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
2089         tmp = REG_SET_FIELD(tmp, CP_PFP_IC_OP_CNTL, INVALIDATE_CACHE, 1);
2090         WREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL, tmp);
2091
2092         /* Wait for invalidation complete */
2093         for (i = 0; i < usec_timeout; i++) {
2094                 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
2095                 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL,
2096                                         INVALIDATE_CACHE_COMPLETE))
2097                         break;
2098                 udelay(1);
2099         }
2100
2101         if (i >= usec_timeout) {
2102                 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2103                 return -EINVAL;
2104         }
2105
2106         if (amdgpu_emu_mode == 1)
2107                 adev->hdp.funcs->flush_hdp(adev, NULL);
2108
2109         tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL);
2110         tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, VMID, 0);
2111         tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, CACHE_POLICY, 0);
2112         tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, EXE_DISABLE, 0);
2113         tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, ADDRESS_CLAMP, 1);
2114         WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL, tmp);
2115
2116         /* Program pfp ucode address into intruction cache address register */
2117         WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_LO,
2118                         lower_32_bits(addr) & 0xFFFFF000);
2119         WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_HI,
2120                         upper_32_bits(addr));
2121
2122         return 0;
2123 }
2124
2125 static int gfx_v11_0_config_mec_cache(struct amdgpu_device *adev, uint64_t addr)
2126 {
2127         uint32_t usec_timeout = 50000;  /* wait for 50ms */
2128         uint32_t tmp;
2129         int i;
2130
2131         /* Trigger an invalidation of the L1 instruction caches */
2132         tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL);
2133         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_OP_CNTL, INVALIDATE_CACHE, 1);
2134
2135         WREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL, tmp);
2136
2137         /* Wait for invalidation complete */
2138         for (i = 0; i < usec_timeout; i++) {
2139                 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL);
2140                 if (1 == REG_GET_FIELD(tmp, CP_CPC_IC_OP_CNTL,
2141                                         INVALIDATE_CACHE_COMPLETE))
2142                         break;
2143                 udelay(1);
2144         }
2145
2146         if (i >= usec_timeout) {
2147                 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2148                 return -EINVAL;
2149         }
2150
2151         if (amdgpu_emu_mode == 1)
2152                 adev->hdp.funcs->flush_hdp(adev, NULL);
2153
2154         tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL);
2155         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0);
2156         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, EXE_DISABLE, 0);
2157         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, ADDRESS_CLAMP, 1);
2158         WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL, tmp);
2159
2160         /* Program mec1 ucode address into intruction cache address register */
2161         WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_LO,
2162                         lower_32_bits(addr) & 0xFFFFF000);
2163         WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_HI,
2164                         upper_32_bits(addr));
2165
2166         return 0;
2167 }
2168
2169 static int gfx_v11_0_config_pfp_cache_rs64(struct amdgpu_device *adev, uint64_t addr, uint64_t addr2)
2170 {
2171         uint32_t usec_timeout = 50000;  /* wait for 50ms */
2172         uint32_t tmp;
2173         unsigned i, pipe_id;
2174         const struct gfx_firmware_header_v2_0 *pfp_hdr;
2175
2176         pfp_hdr = (const struct gfx_firmware_header_v2_0 *)
2177                 adev->gfx.pfp_fw->data;
2178
2179         WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_LO,
2180                 lower_32_bits(addr));
2181         WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_HI,
2182                 upper_32_bits(addr));
2183
2184         tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL);
2185         tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, VMID, 0);
2186         tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, CACHE_POLICY, 0);
2187         tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, EXE_DISABLE, 0);
2188         WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL, tmp);
2189
2190         /*
2191          * Programming any of the CP_PFP_IC_BASE registers
2192          * forces invalidation of the ME L1 I$. Wait for the
2193          * invalidation complete
2194          */
2195         for (i = 0; i < usec_timeout; i++) {
2196                 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
2197                 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL,
2198                         INVALIDATE_CACHE_COMPLETE))
2199                         break;
2200                 udelay(1);
2201         }
2202
2203         if (i >= usec_timeout) {
2204                 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2205                 return -EINVAL;
2206         }
2207
2208         /* Prime the L1 instruction caches */
2209         tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
2210         tmp = REG_SET_FIELD(tmp, CP_PFP_IC_OP_CNTL, PRIME_ICACHE, 1);
2211         WREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL, tmp);
2212         /* Waiting for cache primed*/
2213         for (i = 0; i < usec_timeout; i++) {
2214                 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
2215                 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL,
2216                         ICACHE_PRIMED))
2217                         break;
2218                 udelay(1);
2219         }
2220
2221         if (i >= usec_timeout) {
2222                 dev_err(adev->dev, "failed to prime instruction cache\n");
2223                 return -EINVAL;
2224         }
2225
2226         mutex_lock(&adev->srbm_mutex);
2227         for (pipe_id = 0; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) {
2228                 soc21_grbm_select(adev, 0, pipe_id, 0, 0);
2229                 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START,
2230                         (pfp_hdr->ucode_start_addr_hi << 30) |
2231                         (pfp_hdr->ucode_start_addr_lo >> 2));
2232                 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START_HI,
2233                         pfp_hdr->ucode_start_addr_hi >> 2);
2234
2235                 /*
2236                  * Program CP_ME_CNTL to reset given PIPE to take
2237                  * effect of CP_PFP_PRGRM_CNTR_START.
2238                  */
2239                 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL);
2240                 if (pipe_id == 0)
2241                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2242                                         PFP_PIPE0_RESET, 1);
2243                 else
2244                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2245                                         PFP_PIPE1_RESET, 1);
2246                 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2247
2248                 /* Clear pfp pipe0 reset bit. */
2249                 if (pipe_id == 0)
2250                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2251                                         PFP_PIPE0_RESET, 0);
2252                 else
2253                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2254                                         PFP_PIPE1_RESET, 0);
2255                 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2256
2257                 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE0_LO,
2258                         lower_32_bits(addr2));
2259                 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE0_HI,
2260                         upper_32_bits(addr2));
2261         }
2262         soc21_grbm_select(adev, 0, 0, 0, 0);
2263         mutex_unlock(&adev->srbm_mutex);
2264
2265         tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL);
2266         tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, VMID, 0);
2267         tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, CACHE_POLICY, 0);
2268         WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL, tmp);
2269
2270         /* Invalidate the data caches */
2271         tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
2272         tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, INVALIDATE_DCACHE, 1);
2273         WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL, tmp);
2274
2275         for (i = 0; i < usec_timeout; i++) {
2276                 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
2277                 if (1 == REG_GET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL,
2278                         INVALIDATE_DCACHE_COMPLETE))
2279                         break;
2280                 udelay(1);
2281         }
2282
2283         if (i >= usec_timeout) {
2284                 dev_err(adev->dev, "failed to invalidate RS64 data cache\n");
2285                 return -EINVAL;
2286         }
2287
2288         return 0;
2289 }
2290
2291 static int gfx_v11_0_config_me_cache_rs64(struct amdgpu_device *adev, uint64_t addr, uint64_t addr2)
2292 {
2293         uint32_t usec_timeout = 50000;  /* wait for 50ms */
2294         uint32_t tmp;
2295         unsigned i, pipe_id;
2296         const struct gfx_firmware_header_v2_0 *me_hdr;
2297
2298         me_hdr = (const struct gfx_firmware_header_v2_0 *)
2299                 adev->gfx.me_fw->data;
2300
2301         WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_LO,
2302                 lower_32_bits(addr));
2303         WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_HI,
2304                 upper_32_bits(addr));
2305
2306         tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL);
2307         tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, VMID, 0);
2308         tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, CACHE_POLICY, 0);
2309         tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, EXE_DISABLE, 0);
2310         WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL, tmp);
2311
2312         /*
2313          * Programming any of the CP_ME_IC_BASE registers
2314          * forces invalidation of the ME L1 I$. Wait for the
2315          * invalidation complete
2316          */
2317         for (i = 0; i < usec_timeout; i++) {
2318                 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
2319                 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL,
2320                         INVALIDATE_CACHE_COMPLETE))
2321                         break;
2322                 udelay(1);
2323         }
2324
2325         if (i >= usec_timeout) {
2326                 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2327                 return -EINVAL;
2328         }
2329
2330         /* Prime the instruction caches */
2331         tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
2332         tmp = REG_SET_FIELD(tmp, CP_ME_IC_OP_CNTL, PRIME_ICACHE, 1);
2333         WREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL, tmp);
2334
2335         /* Waiting for instruction cache primed*/
2336         for (i = 0; i < usec_timeout; i++) {
2337                 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
2338                 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL,
2339                         ICACHE_PRIMED))
2340                         break;
2341                 udelay(1);
2342         }
2343
2344         if (i >= usec_timeout) {
2345                 dev_err(adev->dev, "failed to prime instruction cache\n");
2346                 return -EINVAL;
2347         }
2348
2349         mutex_lock(&adev->srbm_mutex);
2350         for (pipe_id = 0; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) {
2351                 soc21_grbm_select(adev, 0, pipe_id, 0, 0);
2352                 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START,
2353                         (me_hdr->ucode_start_addr_hi << 30) |
2354                         (me_hdr->ucode_start_addr_lo >> 2) );
2355                 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START_HI,
2356                         me_hdr->ucode_start_addr_hi>>2);
2357
2358                 /*
2359                  * Program CP_ME_CNTL to reset given PIPE to take
2360                  * effect of CP_PFP_PRGRM_CNTR_START.
2361                  */
2362                 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL);
2363                 if (pipe_id == 0)
2364                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2365                                         ME_PIPE0_RESET, 1);
2366                 else
2367                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2368                                         ME_PIPE1_RESET, 1);
2369                 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2370
2371                 /* Clear pfp pipe0 reset bit. */
2372                 if (pipe_id == 0)
2373                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2374                                         ME_PIPE0_RESET, 0);
2375                 else
2376                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2377                                         ME_PIPE1_RESET, 0);
2378                 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2379
2380                 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE1_LO,
2381                         lower_32_bits(addr2));
2382                 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE1_HI,
2383                         upper_32_bits(addr2));
2384         }
2385         soc21_grbm_select(adev, 0, 0, 0, 0);
2386         mutex_unlock(&adev->srbm_mutex);
2387
2388         tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL);
2389         tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, VMID, 0);
2390         tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, CACHE_POLICY, 0);
2391         WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL, tmp);
2392
2393         /* Invalidate the data caches */
2394         tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
2395         tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, INVALIDATE_DCACHE, 1);
2396         WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL, tmp);
2397
2398         for (i = 0; i < usec_timeout; i++) {
2399                 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
2400                 if (1 == REG_GET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL,
2401                         INVALIDATE_DCACHE_COMPLETE))
2402                         break;
2403                 udelay(1);
2404         }
2405
2406         if (i >= usec_timeout) {
2407                 dev_err(adev->dev, "failed to invalidate RS64 data cache\n");
2408                 return -EINVAL;
2409         }
2410
2411         return 0;
2412 }
2413
2414 static int gfx_v11_0_config_mec_cache_rs64(struct amdgpu_device *adev, uint64_t addr, uint64_t addr2)
2415 {
2416         uint32_t usec_timeout = 50000;  /* wait for 50ms */
2417         uint32_t tmp;
2418         unsigned i;
2419         const struct gfx_firmware_header_v2_0 *mec_hdr;
2420
2421         mec_hdr = (const struct gfx_firmware_header_v2_0 *)
2422                 adev->gfx.mec_fw->data;
2423
2424         tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL);
2425         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0);
2426         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, EXE_DISABLE, 0);
2427         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0);
2428         WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL, tmp);
2429
2430         tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_BASE_CNTL);
2431         tmp = REG_SET_FIELD(tmp, CP_MEC_DC_BASE_CNTL, VMID, 0);
2432         tmp = REG_SET_FIELD(tmp, CP_MEC_DC_BASE_CNTL, CACHE_POLICY, 0);
2433         WREG32_SOC15(GC, 0, regCP_MEC_DC_BASE_CNTL, tmp);
2434
2435         mutex_lock(&adev->srbm_mutex);
2436         for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) {
2437                 soc21_grbm_select(adev, 1, i, 0, 0);
2438
2439                 WREG32_SOC15(GC, 0, regCP_MEC_MDBASE_LO, addr2);
2440                 WREG32_SOC15(GC, 0, regCP_MEC_MDBASE_HI,
2441                      upper_32_bits(addr2));
2442
2443                 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START,
2444                                         mec_hdr->ucode_start_addr_lo >> 2 |
2445                                         mec_hdr->ucode_start_addr_hi << 30);
2446                 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START_HI,
2447                                         mec_hdr->ucode_start_addr_hi >> 2);
2448
2449                 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_LO, addr);
2450                 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_HI,
2451                      upper_32_bits(addr));
2452         }
2453         mutex_unlock(&adev->srbm_mutex);
2454         soc21_grbm_select(adev, 0, 0, 0, 0);
2455
2456         /* Trigger an invalidation of the L1 instruction caches */
2457         tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL);
2458         tmp = REG_SET_FIELD(tmp, CP_MEC_DC_OP_CNTL, INVALIDATE_DCACHE, 1);
2459         WREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL, tmp);
2460
2461         /* Wait for invalidation complete */
2462         for (i = 0; i < usec_timeout; i++) {
2463                 tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL);
2464                 if (1 == REG_GET_FIELD(tmp, CP_MEC_DC_OP_CNTL,
2465                                        INVALIDATE_DCACHE_COMPLETE))
2466                         break;
2467                 udelay(1);
2468         }
2469
2470         if (i >= usec_timeout) {
2471                 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2472                 return -EINVAL;
2473         }
2474
2475         /* Trigger an invalidation of the L1 instruction caches */
2476         tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL);
2477         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_OP_CNTL, INVALIDATE_CACHE, 1);
2478         WREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL, tmp);
2479
2480         /* Wait for invalidation complete */
2481         for (i = 0; i < usec_timeout; i++) {
2482                 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL);
2483                 if (1 == REG_GET_FIELD(tmp, CP_CPC_IC_OP_CNTL,
2484                                        INVALIDATE_CACHE_COMPLETE))
2485                         break;
2486                 udelay(1);
2487         }
2488
2489         if (i >= usec_timeout) {
2490                 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2491                 return -EINVAL;
2492         }
2493
2494         return 0;
2495 }
2496
2497 static void gfx_v11_0_config_gfx_rs64(struct amdgpu_device *adev)
2498 {
2499         const struct gfx_firmware_header_v2_0 *pfp_hdr;
2500         const struct gfx_firmware_header_v2_0 *me_hdr;
2501         const struct gfx_firmware_header_v2_0 *mec_hdr;
2502         uint32_t pipe_id, tmp;
2503
2504         mec_hdr = (const struct gfx_firmware_header_v2_0 *)
2505                 adev->gfx.mec_fw->data;
2506         me_hdr = (const struct gfx_firmware_header_v2_0 *)
2507                 adev->gfx.me_fw->data;
2508         pfp_hdr = (const struct gfx_firmware_header_v2_0 *)
2509                 adev->gfx.pfp_fw->data;
2510
2511         /* config pfp program start addr */
2512         for (pipe_id = 0; pipe_id < 2; pipe_id++) {
2513                 soc21_grbm_select(adev, 0, pipe_id, 0, 0);
2514                 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START,
2515                         (pfp_hdr->ucode_start_addr_hi << 30) |
2516                         (pfp_hdr->ucode_start_addr_lo >> 2));
2517                 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START_HI,
2518                         pfp_hdr->ucode_start_addr_hi >> 2);
2519         }
2520         soc21_grbm_select(adev, 0, 0, 0, 0);
2521
2522         /* reset pfp pipe */
2523         tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL);
2524         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE0_RESET, 1);
2525         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE1_RESET, 1);
2526         WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2527
2528         /* clear pfp pipe reset */
2529         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE0_RESET, 0);
2530         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE1_RESET, 0);
2531         WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2532
2533         /* config me program start addr */
2534         for (pipe_id = 0; pipe_id < 2; pipe_id++) {
2535                 soc21_grbm_select(adev, 0, pipe_id, 0, 0);
2536                 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START,
2537                         (me_hdr->ucode_start_addr_hi << 30) |
2538                         (me_hdr->ucode_start_addr_lo >> 2) );
2539                 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START_HI,
2540                         me_hdr->ucode_start_addr_hi>>2);
2541         }
2542         soc21_grbm_select(adev, 0, 0, 0, 0);
2543
2544         /* reset me pipe */
2545         tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL);
2546         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE0_RESET, 1);
2547         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE1_RESET, 1);
2548         WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2549
2550         /* clear me pipe reset */
2551         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE0_RESET, 0);
2552         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE1_RESET, 0);
2553         WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2554
2555         /* config mec program start addr */
2556         for (pipe_id = 0; pipe_id < 4; pipe_id++) {
2557                 soc21_grbm_select(adev, 1, pipe_id, 0, 0);
2558                 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START,
2559                                         mec_hdr->ucode_start_addr_lo >> 2 |
2560                                         mec_hdr->ucode_start_addr_hi << 30);
2561                 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START_HI,
2562                                         mec_hdr->ucode_start_addr_hi >> 2);
2563         }
2564         soc21_grbm_select(adev, 0, 0, 0, 0);
2565
2566         /* reset mec pipe */
2567         tmp = RREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL);
2568         tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE0_RESET, 1);
2569         tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE1_RESET, 1);
2570         tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE2_RESET, 1);
2571         tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE3_RESET, 1);
2572         WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, tmp);
2573
2574         /* clear mec pipe reset */
2575         tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE0_RESET, 0);
2576         tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE1_RESET, 0);
2577         tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE2_RESET, 0);
2578         tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE3_RESET, 0);
2579         WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, tmp);
2580 }
2581
2582 static int gfx_v11_0_wait_for_rlc_autoload_complete(struct amdgpu_device *adev)
2583 {
2584         uint32_t cp_status;
2585         uint32_t bootload_status;
2586         int i, r;
2587         uint64_t addr, addr2;
2588
2589         for (i = 0; i < adev->usec_timeout; i++) {
2590                 cp_status = RREG32_SOC15(GC, 0, regCP_STAT);
2591
2592                 if (amdgpu_ip_version(adev, GC_HWIP, 0) ==
2593                             IP_VERSION(11, 0, 1) ||
2594                     amdgpu_ip_version(adev, GC_HWIP, 0) ==
2595                             IP_VERSION(11, 0, 4) ||
2596                     amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 5, 0))
2597                         bootload_status = RREG32_SOC15(GC, 0,
2598                                         regRLC_RLCS_BOOTLOAD_STATUS_gc_11_0_1);
2599                 else
2600                         bootload_status = RREG32_SOC15(GC, 0, regRLC_RLCS_BOOTLOAD_STATUS);
2601
2602                 if ((cp_status == 0) &&
2603                     (REG_GET_FIELD(bootload_status,
2604                         RLC_RLCS_BOOTLOAD_STATUS, BOOTLOAD_COMPLETE) == 1)) {
2605                         break;
2606                 }
2607                 udelay(1);
2608         }
2609
2610         if (i >= adev->usec_timeout) {
2611                 dev_err(adev->dev, "rlc autoload: gc ucode autoload timeout\n");
2612                 return -ETIMEDOUT;
2613         }
2614
2615         if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) {
2616                 if (adev->gfx.rs64_enable) {
2617                         addr = adev->gfx.rlc.rlc_autoload_gpu_addr +
2618                                 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_ME].offset;
2619                         addr2 = adev->gfx.rlc.rlc_autoload_gpu_addr +
2620                                 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_ME_P0_STACK].offset;
2621                         r = gfx_v11_0_config_me_cache_rs64(adev, addr, addr2);
2622                         if (r)
2623                                 return r;
2624                         addr = adev->gfx.rlc.rlc_autoload_gpu_addr +
2625                                 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_PFP].offset;
2626                         addr2 = adev->gfx.rlc.rlc_autoload_gpu_addr +
2627                                 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_PFP_P0_STACK].offset;
2628                         r = gfx_v11_0_config_pfp_cache_rs64(adev, addr, addr2);
2629                         if (r)
2630                                 return r;
2631                         addr = adev->gfx.rlc.rlc_autoload_gpu_addr +
2632                                 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_MEC].offset;
2633                         addr2 = adev->gfx.rlc.rlc_autoload_gpu_addr +
2634                                 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_MEC_P0_STACK].offset;
2635                         r = gfx_v11_0_config_mec_cache_rs64(adev, addr, addr2);
2636                         if (r)
2637                                 return r;
2638                 } else {
2639                         addr = adev->gfx.rlc.rlc_autoload_gpu_addr +
2640                                 rlc_autoload_info[SOC21_FIRMWARE_ID_CP_ME].offset;
2641                         r = gfx_v11_0_config_me_cache(adev, addr);
2642                         if (r)
2643                                 return r;
2644                         addr = adev->gfx.rlc.rlc_autoload_gpu_addr +
2645                                 rlc_autoload_info[SOC21_FIRMWARE_ID_CP_PFP].offset;
2646                         r = gfx_v11_0_config_pfp_cache(adev, addr);
2647                         if (r)
2648                                 return r;
2649                         addr = adev->gfx.rlc.rlc_autoload_gpu_addr +
2650                                 rlc_autoload_info[SOC21_FIRMWARE_ID_CP_MEC].offset;
2651                         r = gfx_v11_0_config_mec_cache(adev, addr);
2652                         if (r)
2653                                 return r;
2654                 }
2655         }
2656
2657         return 0;
2658 }
2659
2660 static int gfx_v11_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable)
2661 {
2662         int i;
2663         u32 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL);
2664
2665         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, enable ? 0 : 1);
2666         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, enable ? 0 : 1);
2667         WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2668
2669         for (i = 0; i < adev->usec_timeout; i++) {
2670                 if (RREG32_SOC15(GC, 0, regCP_STAT) == 0)
2671                         break;
2672                 udelay(1);
2673         }
2674
2675         if (i >= adev->usec_timeout)
2676                 DRM_ERROR("failed to %s cp gfx\n", enable ? "unhalt" : "halt");
2677
2678         return 0;
2679 }
2680
2681 static int gfx_v11_0_cp_gfx_load_pfp_microcode(struct amdgpu_device *adev)
2682 {
2683         int r;
2684         const struct gfx_firmware_header_v1_0 *pfp_hdr;
2685         const __le32 *fw_data;
2686         unsigned i, fw_size;
2687
2688         pfp_hdr = (const struct gfx_firmware_header_v1_0 *)
2689                 adev->gfx.pfp_fw->data;
2690
2691         amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header);
2692
2693         fw_data = (const __le32 *)(adev->gfx.pfp_fw->data +
2694                 le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
2695         fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes);
2696
2697         r = amdgpu_bo_create_reserved(adev, pfp_hdr->header.ucode_size_bytes,
2698                                       PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
2699                                       &adev->gfx.pfp.pfp_fw_obj,
2700                                       &adev->gfx.pfp.pfp_fw_gpu_addr,
2701                                       (void **)&adev->gfx.pfp.pfp_fw_ptr);
2702         if (r) {
2703                 dev_err(adev->dev, "(%d) failed to create pfp fw bo\n", r);
2704                 gfx_v11_0_pfp_fini(adev);
2705                 return r;
2706         }
2707
2708         memcpy(adev->gfx.pfp.pfp_fw_ptr, fw_data, fw_size);
2709
2710         amdgpu_bo_kunmap(adev->gfx.pfp.pfp_fw_obj);
2711         amdgpu_bo_unreserve(adev->gfx.pfp.pfp_fw_obj);
2712
2713         gfx_v11_0_config_pfp_cache(adev, adev->gfx.pfp.pfp_fw_gpu_addr);
2714
2715         WREG32_SOC15(GC, 0, regCP_HYP_PFP_UCODE_ADDR, 0);
2716
2717         for (i = 0; i < pfp_hdr->jt_size; i++)
2718                 WREG32_SOC15(GC, 0, regCP_HYP_PFP_UCODE_DATA,
2719                              le32_to_cpup(fw_data + pfp_hdr->jt_offset + i));
2720
2721         WREG32_SOC15(GC, 0, regCP_HYP_PFP_UCODE_ADDR, adev->gfx.pfp_fw_version);
2722
2723         return 0;
2724 }
2725
2726 static int gfx_v11_0_cp_gfx_load_pfp_microcode_rs64(struct amdgpu_device *adev)
2727 {
2728         int r;
2729         const struct gfx_firmware_header_v2_0 *pfp_hdr;
2730         const __le32 *fw_ucode, *fw_data;
2731         unsigned i, pipe_id, fw_ucode_size, fw_data_size;
2732         uint32_t tmp;
2733         uint32_t usec_timeout = 50000;  /* wait for 50ms */
2734
2735         pfp_hdr = (const struct gfx_firmware_header_v2_0 *)
2736                 adev->gfx.pfp_fw->data;
2737
2738         amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header);
2739
2740         /* instruction */
2741         fw_ucode = (const __le32 *)(adev->gfx.pfp_fw->data +
2742                 le32_to_cpu(pfp_hdr->ucode_offset_bytes));
2743         fw_ucode_size = le32_to_cpu(pfp_hdr->ucode_size_bytes);
2744         /* data */
2745         fw_data = (const __le32 *)(adev->gfx.pfp_fw->data +
2746                 le32_to_cpu(pfp_hdr->data_offset_bytes));
2747         fw_data_size = le32_to_cpu(pfp_hdr->data_size_bytes);
2748
2749         /* 64kb align */
2750         r = amdgpu_bo_create_reserved(adev, fw_ucode_size,
2751                                       64 * 1024,
2752                                       AMDGPU_GEM_DOMAIN_VRAM |
2753                                       AMDGPU_GEM_DOMAIN_GTT,
2754                                       &adev->gfx.pfp.pfp_fw_obj,
2755                                       &adev->gfx.pfp.pfp_fw_gpu_addr,
2756                                       (void **)&adev->gfx.pfp.pfp_fw_ptr);
2757         if (r) {
2758                 dev_err(adev->dev, "(%d) failed to create pfp ucode fw bo\n", r);
2759                 gfx_v11_0_pfp_fini(adev);
2760                 return r;
2761         }
2762
2763         r = amdgpu_bo_create_reserved(adev, fw_data_size,
2764                                       64 * 1024,
2765                                       AMDGPU_GEM_DOMAIN_VRAM |
2766                                       AMDGPU_GEM_DOMAIN_GTT,
2767                                       &adev->gfx.pfp.pfp_fw_data_obj,
2768                                       &adev->gfx.pfp.pfp_fw_data_gpu_addr,
2769                                       (void **)&adev->gfx.pfp.pfp_fw_data_ptr);
2770         if (r) {
2771                 dev_err(adev->dev, "(%d) failed to create pfp data fw bo\n", r);
2772                 gfx_v11_0_pfp_fini(adev);
2773                 return r;
2774         }
2775
2776         memcpy(adev->gfx.pfp.pfp_fw_ptr, fw_ucode, fw_ucode_size);
2777         memcpy(adev->gfx.pfp.pfp_fw_data_ptr, fw_data, fw_data_size);
2778
2779         amdgpu_bo_kunmap(adev->gfx.pfp.pfp_fw_obj);
2780         amdgpu_bo_kunmap(adev->gfx.pfp.pfp_fw_data_obj);
2781         amdgpu_bo_unreserve(adev->gfx.pfp.pfp_fw_obj);
2782         amdgpu_bo_unreserve(adev->gfx.pfp.pfp_fw_data_obj);
2783
2784         if (amdgpu_emu_mode == 1)
2785                 adev->hdp.funcs->flush_hdp(adev, NULL);
2786
2787         WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_LO,
2788                 lower_32_bits(adev->gfx.pfp.pfp_fw_gpu_addr));
2789         WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_HI,
2790                 upper_32_bits(adev->gfx.pfp.pfp_fw_gpu_addr));
2791
2792         tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL);
2793         tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, VMID, 0);
2794         tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, CACHE_POLICY, 0);
2795         tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, EXE_DISABLE, 0);
2796         WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL, tmp);
2797
2798         /*
2799          * Programming any of the CP_PFP_IC_BASE registers
2800          * forces invalidation of the ME L1 I$. Wait for the
2801          * invalidation complete
2802          */
2803         for (i = 0; i < usec_timeout; i++) {
2804                 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
2805                 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL,
2806                         INVALIDATE_CACHE_COMPLETE))
2807                         break;
2808                 udelay(1);
2809         }
2810
2811         if (i >= usec_timeout) {
2812                 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2813                 return -EINVAL;
2814         }
2815
2816         /* Prime the L1 instruction caches */
2817         tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
2818         tmp = REG_SET_FIELD(tmp, CP_PFP_IC_OP_CNTL, PRIME_ICACHE, 1);
2819         WREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL, tmp);
2820         /* Waiting for cache primed*/
2821         for (i = 0; i < usec_timeout; i++) {
2822                 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
2823                 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL,
2824                         ICACHE_PRIMED))
2825                         break;
2826                 udelay(1);
2827         }
2828
2829         if (i >= usec_timeout) {
2830                 dev_err(adev->dev, "failed to prime instruction cache\n");
2831                 return -EINVAL;
2832         }
2833
2834         mutex_lock(&adev->srbm_mutex);
2835         for (pipe_id = 0; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) {
2836                 soc21_grbm_select(adev, 0, pipe_id, 0, 0);
2837                 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START,
2838                         (pfp_hdr->ucode_start_addr_hi << 30) |
2839                         (pfp_hdr->ucode_start_addr_lo >> 2) );
2840                 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START_HI,
2841                         pfp_hdr->ucode_start_addr_hi>>2);
2842
2843                 /*
2844                  * Program CP_ME_CNTL to reset given PIPE to take
2845                  * effect of CP_PFP_PRGRM_CNTR_START.
2846                  */
2847                 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL);
2848                 if (pipe_id == 0)
2849                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2850                                         PFP_PIPE0_RESET, 1);
2851                 else
2852                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2853                                         PFP_PIPE1_RESET, 1);
2854                 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2855
2856                 /* Clear pfp pipe0 reset bit. */
2857                 if (pipe_id == 0)
2858                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2859                                         PFP_PIPE0_RESET, 0);
2860                 else
2861                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2862                                         PFP_PIPE1_RESET, 0);
2863                 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2864
2865                 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE0_LO,
2866                         lower_32_bits(adev->gfx.pfp.pfp_fw_data_gpu_addr));
2867                 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE0_HI,
2868                         upper_32_bits(adev->gfx.pfp.pfp_fw_data_gpu_addr));
2869         }
2870         soc21_grbm_select(adev, 0, 0, 0, 0);
2871         mutex_unlock(&adev->srbm_mutex);
2872
2873         tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL);
2874         tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, VMID, 0);
2875         tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, CACHE_POLICY, 0);
2876         WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL, tmp);
2877
2878         /* Invalidate the data caches */
2879         tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
2880         tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, INVALIDATE_DCACHE, 1);
2881         WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL, tmp);
2882
2883         for (i = 0; i < usec_timeout; i++) {
2884                 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
2885                 if (1 == REG_GET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL,
2886                         INVALIDATE_DCACHE_COMPLETE))
2887                         break;
2888                 udelay(1);
2889         }
2890
2891         if (i >= usec_timeout) {
2892                 dev_err(adev->dev, "failed to invalidate RS64 data cache\n");
2893                 return -EINVAL;
2894         }
2895
2896         return 0;
2897 }
2898
2899 static int gfx_v11_0_cp_gfx_load_me_microcode(struct amdgpu_device *adev)
2900 {
2901         int r;
2902         const struct gfx_firmware_header_v1_0 *me_hdr;
2903         const __le32 *fw_data;
2904         unsigned i, fw_size;
2905
2906         me_hdr = (const struct gfx_firmware_header_v1_0 *)
2907                 adev->gfx.me_fw->data;
2908
2909         amdgpu_ucode_print_gfx_hdr(&me_hdr->header);
2910
2911         fw_data = (const __le32 *)(adev->gfx.me_fw->data +
2912                 le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
2913         fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes);
2914
2915         r = amdgpu_bo_create_reserved(adev, me_hdr->header.ucode_size_bytes,
2916                                       PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
2917                                       &adev->gfx.me.me_fw_obj,
2918                                       &adev->gfx.me.me_fw_gpu_addr,
2919                                       (void **)&adev->gfx.me.me_fw_ptr);
2920         if (r) {
2921                 dev_err(adev->dev, "(%d) failed to create me fw bo\n", r);
2922                 gfx_v11_0_me_fini(adev);
2923                 return r;
2924         }
2925
2926         memcpy(adev->gfx.me.me_fw_ptr, fw_data, fw_size);
2927
2928         amdgpu_bo_kunmap(adev->gfx.me.me_fw_obj);
2929         amdgpu_bo_unreserve(adev->gfx.me.me_fw_obj);
2930
2931         gfx_v11_0_config_me_cache(adev, adev->gfx.me.me_fw_gpu_addr);
2932
2933         WREG32_SOC15(GC, 0, regCP_HYP_ME_UCODE_ADDR, 0);
2934
2935         for (i = 0; i < me_hdr->jt_size; i++)
2936                 WREG32_SOC15(GC, 0, regCP_HYP_ME_UCODE_DATA,
2937                              le32_to_cpup(fw_data + me_hdr->jt_offset + i));
2938
2939         WREG32_SOC15(GC, 0, regCP_HYP_ME_UCODE_ADDR, adev->gfx.me_fw_version);
2940
2941         return 0;
2942 }
2943
2944 static int gfx_v11_0_cp_gfx_load_me_microcode_rs64(struct amdgpu_device *adev)
2945 {
2946         int r;
2947         const struct gfx_firmware_header_v2_0 *me_hdr;
2948         const __le32 *fw_ucode, *fw_data;
2949         unsigned i, pipe_id, fw_ucode_size, fw_data_size;
2950         uint32_t tmp;
2951         uint32_t usec_timeout = 50000;  /* wait for 50ms */
2952
2953         me_hdr = (const struct gfx_firmware_header_v2_0 *)
2954                 adev->gfx.me_fw->data;
2955
2956         amdgpu_ucode_print_gfx_hdr(&me_hdr->header);
2957
2958         /* instruction */
2959         fw_ucode = (const __le32 *)(adev->gfx.me_fw->data +
2960                 le32_to_cpu(me_hdr->ucode_offset_bytes));
2961         fw_ucode_size = le32_to_cpu(me_hdr->ucode_size_bytes);
2962         /* data */
2963         fw_data = (const __le32 *)(adev->gfx.me_fw->data +
2964                 le32_to_cpu(me_hdr->data_offset_bytes));
2965         fw_data_size = le32_to_cpu(me_hdr->data_size_bytes);
2966
2967         /* 64kb align*/
2968         r = amdgpu_bo_create_reserved(adev, fw_ucode_size,
2969                                       64 * 1024,
2970                                       AMDGPU_GEM_DOMAIN_VRAM |
2971                                       AMDGPU_GEM_DOMAIN_GTT,
2972                                       &adev->gfx.me.me_fw_obj,
2973                                       &adev->gfx.me.me_fw_gpu_addr,
2974                                       (void **)&adev->gfx.me.me_fw_ptr);
2975         if (r) {
2976                 dev_err(adev->dev, "(%d) failed to create me ucode bo\n", r);
2977                 gfx_v11_0_me_fini(adev);
2978                 return r;
2979         }
2980
2981         r = amdgpu_bo_create_reserved(adev, fw_data_size,
2982                                       64 * 1024,
2983                                       AMDGPU_GEM_DOMAIN_VRAM |
2984                                       AMDGPU_GEM_DOMAIN_GTT,
2985                                       &adev->gfx.me.me_fw_data_obj,
2986                                       &adev->gfx.me.me_fw_data_gpu_addr,
2987                                       (void **)&adev->gfx.me.me_fw_data_ptr);
2988         if (r) {
2989                 dev_err(adev->dev, "(%d) failed to create me data bo\n", r);
2990                 gfx_v11_0_pfp_fini(adev);
2991                 return r;
2992         }
2993
2994         memcpy(adev->gfx.me.me_fw_ptr, fw_ucode, fw_ucode_size);
2995         memcpy(adev->gfx.me.me_fw_data_ptr, fw_data, fw_data_size);
2996
2997         amdgpu_bo_kunmap(adev->gfx.me.me_fw_obj);
2998         amdgpu_bo_kunmap(adev->gfx.me.me_fw_data_obj);
2999         amdgpu_bo_unreserve(adev->gfx.me.me_fw_obj);
3000         amdgpu_bo_unreserve(adev->gfx.me.me_fw_data_obj);
3001
3002         if (amdgpu_emu_mode == 1)
3003                 adev->hdp.funcs->flush_hdp(adev, NULL);
3004
3005         WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_LO,
3006                 lower_32_bits(adev->gfx.me.me_fw_gpu_addr));
3007         WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_HI,
3008                 upper_32_bits(adev->gfx.me.me_fw_gpu_addr));
3009
3010         tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL);
3011         tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, VMID, 0);
3012         tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, CACHE_POLICY, 0);
3013         tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, EXE_DISABLE, 0);
3014         WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL, tmp);
3015
3016         /*
3017          * Programming any of the CP_ME_IC_BASE registers
3018          * forces invalidation of the ME L1 I$. Wait for the
3019          * invalidation complete
3020          */
3021         for (i = 0; i < usec_timeout; i++) {
3022                 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
3023                 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL,
3024                         INVALIDATE_CACHE_COMPLETE))
3025                         break;
3026                 udelay(1);
3027         }
3028
3029         if (i >= usec_timeout) {
3030                 dev_err(adev->dev, "failed to invalidate instruction cache\n");
3031                 return -EINVAL;
3032         }
3033
3034         /* Prime the instruction caches */
3035         tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
3036         tmp = REG_SET_FIELD(tmp, CP_ME_IC_OP_CNTL, PRIME_ICACHE, 1);
3037         WREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL, tmp);
3038
3039         /* Waiting for instruction cache primed*/
3040         for (i = 0; i < usec_timeout; i++) {
3041                 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
3042                 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL,
3043                         ICACHE_PRIMED))
3044                         break;
3045                 udelay(1);
3046         }
3047
3048         if (i >= usec_timeout) {
3049                 dev_err(adev->dev, "failed to prime instruction cache\n");
3050                 return -EINVAL;
3051         }
3052
3053         mutex_lock(&adev->srbm_mutex);
3054         for (pipe_id = 0; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) {
3055                 soc21_grbm_select(adev, 0, pipe_id, 0, 0);
3056                 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START,
3057                         (me_hdr->ucode_start_addr_hi << 30) |
3058                         (me_hdr->ucode_start_addr_lo >> 2) );
3059                 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START_HI,
3060                         me_hdr->ucode_start_addr_hi>>2);
3061
3062                 /*
3063                  * Program CP_ME_CNTL to reset given PIPE to take
3064                  * effect of CP_PFP_PRGRM_CNTR_START.
3065                  */
3066                 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL);
3067                 if (pipe_id == 0)
3068                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
3069                                         ME_PIPE0_RESET, 1);
3070                 else
3071                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
3072                                         ME_PIPE1_RESET, 1);
3073                 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
3074
3075                 /* Clear pfp pipe0 reset bit. */
3076                 if (pipe_id == 0)
3077                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
3078                                         ME_PIPE0_RESET, 0);
3079                 else
3080                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
3081                                         ME_PIPE1_RESET, 0);
3082                 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
3083
3084                 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE1_LO,
3085                         lower_32_bits(adev->gfx.me.me_fw_data_gpu_addr));
3086                 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE1_HI,
3087                         upper_32_bits(adev->gfx.me.me_fw_data_gpu_addr));
3088         }
3089         soc21_grbm_select(adev, 0, 0, 0, 0);
3090         mutex_unlock(&adev->srbm_mutex);
3091
3092         tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL);
3093         tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, VMID, 0);
3094         tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, CACHE_POLICY, 0);
3095         WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL, tmp);
3096
3097         /* Invalidate the data caches */
3098         tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
3099         tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, INVALIDATE_DCACHE, 1);
3100         WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL, tmp);
3101
3102         for (i = 0; i < usec_timeout; i++) {
3103                 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
3104                 if (1 == REG_GET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL,
3105                         INVALIDATE_DCACHE_COMPLETE))
3106                         break;
3107                 udelay(1);
3108         }
3109
3110         if (i >= usec_timeout) {
3111                 dev_err(adev->dev, "failed to invalidate RS64 data cache\n");
3112                 return -EINVAL;
3113         }
3114
3115         return 0;
3116 }
3117
3118 static int gfx_v11_0_cp_gfx_load_microcode(struct amdgpu_device *adev)
3119 {
3120         int r;
3121
3122         if (!adev->gfx.me_fw || !adev->gfx.pfp_fw)
3123                 return -EINVAL;
3124
3125         gfx_v11_0_cp_gfx_enable(adev, false);
3126
3127         if (adev->gfx.rs64_enable)
3128                 r = gfx_v11_0_cp_gfx_load_pfp_microcode_rs64(adev);
3129         else
3130                 r = gfx_v11_0_cp_gfx_load_pfp_microcode(adev);
3131         if (r) {
3132                 dev_err(adev->dev, "(%d) failed to load pfp fw\n", r);
3133                 return r;
3134         }
3135
3136         if (adev->gfx.rs64_enable)
3137                 r = gfx_v11_0_cp_gfx_load_me_microcode_rs64(adev);
3138         else
3139                 r = gfx_v11_0_cp_gfx_load_me_microcode(adev);
3140         if (r) {
3141                 dev_err(adev->dev, "(%d) failed to load me fw\n", r);
3142                 return r;
3143         }
3144
3145         return 0;
3146 }
3147
3148 static int gfx_v11_0_cp_gfx_start(struct amdgpu_device *adev)
3149 {
3150         struct amdgpu_ring *ring;
3151         const struct cs_section_def *sect = NULL;
3152         const struct cs_extent_def *ext = NULL;
3153         int r, i;
3154         int ctx_reg_offset;
3155
3156         /* init the CP */
3157         WREG32_SOC15(GC, 0, regCP_MAX_CONTEXT,
3158                      adev->gfx.config.max_hw_contexts - 1);
3159         WREG32_SOC15(GC, 0, regCP_DEVICE_ID, 1);
3160
3161         if (!amdgpu_async_gfx_ring)
3162                 gfx_v11_0_cp_gfx_enable(adev, true);
3163
3164         ring = &adev->gfx.gfx_ring[0];
3165         r = amdgpu_ring_alloc(ring, gfx_v11_0_get_csb_size(adev));
3166         if (r) {
3167                 DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r);
3168                 return r;
3169         }
3170
3171         amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3172         amdgpu_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3173
3174         amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
3175         amdgpu_ring_write(ring, 0x80000000);
3176         amdgpu_ring_write(ring, 0x80000000);
3177
3178         for (sect = gfx11_cs_data; sect->section != NULL; ++sect) {
3179                 for (ext = sect->section; ext->extent != NULL; ++ext) {
3180                         if (sect->id == SECT_CONTEXT) {
3181                                 amdgpu_ring_write(ring,
3182                                                   PACKET3(PACKET3_SET_CONTEXT_REG,
3183                                                           ext->reg_count));
3184                                 amdgpu_ring_write(ring, ext->reg_index -
3185                                                   PACKET3_SET_CONTEXT_REG_START);
3186                                 for (i = 0; i < ext->reg_count; i++)
3187                                         amdgpu_ring_write(ring, ext->extent[i]);
3188                         }
3189                 }
3190         }
3191
3192         ctx_reg_offset =
3193                 SOC15_REG_OFFSET(GC, 0, regPA_SC_TILE_STEERING_OVERRIDE) - PACKET3_SET_CONTEXT_REG_START;
3194         amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 1));
3195         amdgpu_ring_write(ring, ctx_reg_offset);
3196         amdgpu_ring_write(ring, adev->gfx.config.pa_sc_tile_steering_override);
3197
3198         amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3199         amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3200
3201         amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3202         amdgpu_ring_write(ring, 0);
3203
3204         amdgpu_ring_commit(ring);
3205
3206         /* submit cs packet to copy state 0 to next available state */
3207         if (adev->gfx.num_gfx_rings > 1) {
3208                 /* maximum supported gfx ring is 2 */
3209                 ring = &adev->gfx.gfx_ring[1];
3210                 r = amdgpu_ring_alloc(ring, 2);
3211                 if (r) {
3212                         DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r);
3213                         return r;
3214                 }
3215
3216                 amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3217                 amdgpu_ring_write(ring, 0);
3218
3219                 amdgpu_ring_commit(ring);
3220         }
3221         return 0;
3222 }
3223
3224 static void gfx_v11_0_cp_gfx_switch_pipe(struct amdgpu_device *adev,
3225                                          CP_PIPE_ID pipe)
3226 {
3227         u32 tmp;
3228
3229         tmp = RREG32_SOC15(GC, 0, regGRBM_GFX_CNTL);
3230         tmp = REG_SET_FIELD(tmp, GRBM_GFX_CNTL, PIPEID, pipe);
3231
3232         WREG32_SOC15(GC, 0, regGRBM_GFX_CNTL, tmp);
3233 }
3234
3235 static void gfx_v11_0_cp_gfx_set_doorbell(struct amdgpu_device *adev,
3236                                           struct amdgpu_ring *ring)
3237 {
3238         u32 tmp;
3239
3240         tmp = RREG32_SOC15(GC, 0, regCP_RB_DOORBELL_CONTROL);
3241         if (ring->use_doorbell) {
3242                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3243                                     DOORBELL_OFFSET, ring->doorbell_index);
3244                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3245                                     DOORBELL_EN, 1);
3246         } else {
3247                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3248                                     DOORBELL_EN, 0);
3249         }
3250         WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_CONTROL, tmp);
3251
3252         tmp = REG_SET_FIELD(0, CP_RB_DOORBELL_RANGE_LOWER,
3253                             DOORBELL_RANGE_LOWER, ring->doorbell_index);
3254         WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_RANGE_LOWER, tmp);
3255
3256         WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_RANGE_UPPER,
3257                      CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK);
3258 }
3259
3260 static int gfx_v11_0_cp_gfx_resume(struct amdgpu_device *adev)
3261 {
3262         struct amdgpu_ring *ring;
3263         u32 tmp;
3264         u32 rb_bufsz;
3265         u64 rb_addr, rptr_addr, wptr_gpu_addr;
3266
3267         /* Set the write pointer delay */
3268         WREG32_SOC15(GC, 0, regCP_RB_WPTR_DELAY, 0);
3269
3270         /* set the RB to use vmid 0 */
3271         WREG32_SOC15(GC, 0, regCP_RB_VMID, 0);
3272
3273         /* Init gfx ring 0 for pipe 0 */
3274         mutex_lock(&adev->srbm_mutex);
3275         gfx_v11_0_cp_gfx_switch_pipe(adev, PIPE_ID0);
3276
3277         /* Set ring buffer size */
3278         ring = &adev->gfx.gfx_ring[0];
3279         rb_bufsz = order_base_2(ring->ring_size / 8);
3280         tmp = REG_SET_FIELD(0, CP_RB0_CNTL, RB_BUFSZ, rb_bufsz);
3281         tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, RB_BLKSZ, rb_bufsz - 2);
3282         WREG32_SOC15(GC, 0, regCP_RB0_CNTL, tmp);
3283
3284         /* Initialize the ring buffer's write pointers */
3285         ring->wptr = 0;
3286         WREG32_SOC15(GC, 0, regCP_RB0_WPTR, lower_32_bits(ring->wptr));
3287         WREG32_SOC15(GC, 0, regCP_RB0_WPTR_HI, upper_32_bits(ring->wptr));
3288
3289         /* set the wb address wether it's enabled or not */
3290         rptr_addr = ring->rptr_gpu_addr;
3291         WREG32_SOC15(GC, 0, regCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr));
3292         WREG32_SOC15(GC, 0, regCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) &
3293                      CP_RB_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK);
3294
3295         wptr_gpu_addr = ring->wptr_gpu_addr;
3296         WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_ADDR_LO,
3297                      lower_32_bits(wptr_gpu_addr));
3298         WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_ADDR_HI,
3299                      upper_32_bits(wptr_gpu_addr));
3300
3301         mdelay(1);
3302         WREG32_SOC15(GC, 0, regCP_RB0_CNTL, tmp);
3303
3304         rb_addr = ring->gpu_addr >> 8;
3305         WREG32_SOC15(GC, 0, regCP_RB0_BASE, rb_addr);
3306         WREG32_SOC15(GC, 0, regCP_RB0_BASE_HI, upper_32_bits(rb_addr));
3307
3308         WREG32_SOC15(GC, 0, regCP_RB_ACTIVE, 1);
3309
3310         gfx_v11_0_cp_gfx_set_doorbell(adev, ring);
3311         mutex_unlock(&adev->srbm_mutex);
3312
3313         /* Init gfx ring 1 for pipe 1 */
3314         if (adev->gfx.num_gfx_rings > 1) {
3315                 mutex_lock(&adev->srbm_mutex);
3316                 gfx_v11_0_cp_gfx_switch_pipe(adev, PIPE_ID1);
3317                 /* maximum supported gfx ring is 2 */
3318                 ring = &adev->gfx.gfx_ring[1];
3319                 rb_bufsz = order_base_2(ring->ring_size / 8);
3320                 tmp = REG_SET_FIELD(0, CP_RB1_CNTL, RB_BUFSZ, rb_bufsz);
3321                 tmp = REG_SET_FIELD(tmp, CP_RB1_CNTL, RB_BLKSZ, rb_bufsz - 2);
3322                 WREG32_SOC15(GC, 0, regCP_RB1_CNTL, tmp);
3323                 /* Initialize the ring buffer's write pointers */
3324                 ring->wptr = 0;
3325                 WREG32_SOC15(GC, 0, regCP_RB1_WPTR, lower_32_bits(ring->wptr));
3326                 WREG32_SOC15(GC, 0, regCP_RB1_WPTR_HI, upper_32_bits(ring->wptr));
3327                 /* Set the wb address wether it's enabled or not */
3328                 rptr_addr = ring->rptr_gpu_addr;
3329                 WREG32_SOC15(GC, 0, regCP_RB1_RPTR_ADDR, lower_32_bits(rptr_addr));
3330                 WREG32_SOC15(GC, 0, regCP_RB1_RPTR_ADDR_HI, upper_32_bits(rptr_addr) &
3331                              CP_RB1_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK);
3332                 wptr_gpu_addr = ring->wptr_gpu_addr;
3333                 WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_ADDR_LO,
3334                              lower_32_bits(wptr_gpu_addr));
3335                 WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_ADDR_HI,
3336                              upper_32_bits(wptr_gpu_addr));
3337
3338                 mdelay(1);
3339                 WREG32_SOC15(GC, 0, regCP_RB1_CNTL, tmp);
3340
3341                 rb_addr = ring->gpu_addr >> 8;
3342                 WREG32_SOC15(GC, 0, regCP_RB1_BASE, rb_addr);
3343                 WREG32_SOC15(GC, 0, regCP_RB1_BASE_HI, upper_32_bits(rb_addr));
3344                 WREG32_SOC15(GC, 0, regCP_RB1_ACTIVE, 1);
3345
3346                 gfx_v11_0_cp_gfx_set_doorbell(adev, ring);
3347                 mutex_unlock(&adev->srbm_mutex);
3348         }
3349         /* Switch to pipe 0 */
3350         mutex_lock(&adev->srbm_mutex);
3351         gfx_v11_0_cp_gfx_switch_pipe(adev, PIPE_ID0);
3352         mutex_unlock(&adev->srbm_mutex);
3353
3354         /* start the ring */
3355         gfx_v11_0_cp_gfx_start(adev);
3356
3357         return 0;
3358 }
3359
3360 static void gfx_v11_0_cp_compute_enable(struct amdgpu_device *adev, bool enable)
3361 {
3362         u32 data;
3363
3364         if (adev->gfx.rs64_enable) {
3365                 data = RREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL);
3366                 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_INVALIDATE_ICACHE,
3367                                                          enable ? 0 : 1);
3368                 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE0_RESET,
3369                                                          enable ? 0 : 1);
3370                 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE1_RESET,
3371                                                          enable ? 0 : 1);
3372                 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE2_RESET,
3373                                                          enable ? 0 : 1);
3374                 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE3_RESET,
3375                                                          enable ? 0 : 1);
3376                 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE0_ACTIVE,
3377                                                          enable ? 1 : 0);
3378                 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE1_ACTIVE,
3379                                                          enable ? 1 : 0);
3380                 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE2_ACTIVE,
3381                                                          enable ? 1 : 0);
3382                 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE3_ACTIVE,
3383                                                          enable ? 1 : 0);
3384                 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_HALT,
3385                                                          enable ? 0 : 1);
3386                 WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, data);
3387         } else {
3388                 data = RREG32_SOC15(GC, 0, regCP_MEC_CNTL);
3389
3390                 if (enable) {
3391                         data = REG_SET_FIELD(data, CP_MEC_CNTL, MEC_ME1_HALT, 0);
3392                         if (!adev->enable_mes_kiq)
3393                                 data = REG_SET_FIELD(data, CP_MEC_CNTL,
3394                                                      MEC_ME2_HALT, 0);
3395                 } else {
3396                         data = REG_SET_FIELD(data, CP_MEC_CNTL, MEC_ME1_HALT, 1);
3397                         data = REG_SET_FIELD(data, CP_MEC_CNTL, MEC_ME2_HALT, 1);
3398                 }
3399                 WREG32_SOC15(GC, 0, regCP_MEC_CNTL, data);
3400         }
3401
3402         udelay(50);
3403 }
3404
3405 static int gfx_v11_0_cp_compute_load_microcode(struct amdgpu_device *adev)
3406 {
3407         const struct gfx_firmware_header_v1_0 *mec_hdr;
3408         const __le32 *fw_data;
3409         unsigned i, fw_size;
3410         u32 *fw = NULL;
3411         int r;
3412
3413         if (!adev->gfx.mec_fw)
3414                 return -EINVAL;
3415
3416         gfx_v11_0_cp_compute_enable(adev, false);
3417
3418         mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
3419         amdgpu_ucode_print_gfx_hdr(&mec_hdr->header);
3420
3421         fw_data = (const __le32 *)
3422                 (adev->gfx.mec_fw->data +
3423                  le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
3424         fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes);
3425
3426         r = amdgpu_bo_create_reserved(adev, mec_hdr->header.ucode_size_bytes,
3427                                           PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
3428                                           &adev->gfx.mec.mec_fw_obj,
3429                                           &adev->gfx.mec.mec_fw_gpu_addr,
3430                                           (void **)&fw);
3431         if (r) {
3432                 dev_err(adev->dev, "(%d) failed to create mec fw bo\n", r);
3433                 gfx_v11_0_mec_fini(adev);
3434                 return r;
3435         }
3436
3437         memcpy(fw, fw_data, fw_size);
3438
3439         amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj);
3440         amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj);
3441
3442         gfx_v11_0_config_mec_cache(adev, adev->gfx.mec.mec_fw_gpu_addr);
3443
3444         /* MEC1 */
3445         WREG32_SOC15(GC, 0, regCP_MEC_ME1_UCODE_ADDR, 0);
3446
3447         for (i = 0; i < mec_hdr->jt_size; i++)
3448                 WREG32_SOC15(GC, 0, regCP_MEC_ME1_UCODE_DATA,
3449                              le32_to_cpup(fw_data + mec_hdr->jt_offset + i));
3450
3451         WREG32_SOC15(GC, 0, regCP_MEC_ME1_UCODE_ADDR, adev->gfx.mec_fw_version);
3452
3453         return 0;
3454 }
3455
3456 static int gfx_v11_0_cp_compute_load_microcode_rs64(struct amdgpu_device *adev)
3457 {
3458         const struct gfx_firmware_header_v2_0 *mec_hdr;
3459         const __le32 *fw_ucode, *fw_data;
3460         u32 tmp, fw_ucode_size, fw_data_size;
3461         u32 i, usec_timeout = 50000; /* Wait for 50 ms */
3462         u32 *fw_ucode_ptr, *fw_data_ptr;
3463         int r;
3464
3465         if (!adev->gfx.mec_fw)
3466                 return -EINVAL;
3467
3468         gfx_v11_0_cp_compute_enable(adev, false);
3469
3470         mec_hdr = (const struct gfx_firmware_header_v2_0 *)adev->gfx.mec_fw->data;
3471         amdgpu_ucode_print_gfx_hdr(&mec_hdr->header);
3472
3473         fw_ucode = (const __le32 *) (adev->gfx.mec_fw->data +
3474                                 le32_to_cpu(mec_hdr->ucode_offset_bytes));
3475         fw_ucode_size = le32_to_cpu(mec_hdr->ucode_size_bytes);
3476
3477         fw_data = (const __le32 *) (adev->gfx.mec_fw->data +
3478                                 le32_to_cpu(mec_hdr->data_offset_bytes));
3479         fw_data_size = le32_to_cpu(mec_hdr->data_size_bytes);
3480
3481         r = amdgpu_bo_create_reserved(adev, fw_ucode_size,
3482                                       64 * 1024,
3483                                       AMDGPU_GEM_DOMAIN_VRAM |
3484                                       AMDGPU_GEM_DOMAIN_GTT,
3485                                       &adev->gfx.mec.mec_fw_obj,
3486                                       &adev->gfx.mec.mec_fw_gpu_addr,
3487                                       (void **)&fw_ucode_ptr);
3488         if (r) {
3489                 dev_err(adev->dev, "(%d) failed to create mec fw ucode bo\n", r);
3490                 gfx_v11_0_mec_fini(adev);
3491                 return r;
3492         }
3493
3494         r = amdgpu_bo_create_reserved(adev, fw_data_size,
3495                                       64 * 1024,
3496                                       AMDGPU_GEM_DOMAIN_VRAM |
3497                                       AMDGPU_GEM_DOMAIN_GTT,
3498                                       &adev->gfx.mec.mec_fw_data_obj,
3499                                       &adev->gfx.mec.mec_fw_data_gpu_addr,
3500                                       (void **)&fw_data_ptr);
3501         if (r) {
3502                 dev_err(adev->dev, "(%d) failed to create mec fw ucode bo\n", r);
3503                 gfx_v11_0_mec_fini(adev);
3504                 return r;
3505         }
3506
3507         memcpy(fw_ucode_ptr, fw_ucode, fw_ucode_size);
3508         memcpy(fw_data_ptr, fw_data, fw_data_size);
3509
3510         amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj);
3511         amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_data_obj);
3512         amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj);
3513         amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_data_obj);
3514
3515         tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL);
3516         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0);
3517         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, EXE_DISABLE, 0);
3518         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0);
3519         WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL, tmp);
3520
3521         tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_BASE_CNTL);
3522         tmp = REG_SET_FIELD(tmp, CP_MEC_DC_BASE_CNTL, VMID, 0);
3523         tmp = REG_SET_FIELD(tmp, CP_MEC_DC_BASE_CNTL, CACHE_POLICY, 0);
3524         WREG32_SOC15(GC, 0, regCP_MEC_DC_BASE_CNTL, tmp);
3525
3526         mutex_lock(&adev->srbm_mutex);
3527         for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) {
3528                 soc21_grbm_select(adev, 1, i, 0, 0);
3529
3530                 WREG32_SOC15(GC, 0, regCP_MEC_MDBASE_LO, adev->gfx.mec.mec_fw_data_gpu_addr);
3531                 WREG32_SOC15(GC, 0, regCP_MEC_MDBASE_HI,
3532                      upper_32_bits(adev->gfx.mec.mec_fw_data_gpu_addr));
3533
3534                 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START,
3535                                         mec_hdr->ucode_start_addr_lo >> 2 |
3536                                         mec_hdr->ucode_start_addr_hi << 30);
3537                 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START_HI,
3538                                         mec_hdr->ucode_start_addr_hi >> 2);
3539
3540                 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_LO, adev->gfx.mec.mec_fw_gpu_addr);
3541                 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_HI,
3542                      upper_32_bits(adev->gfx.mec.mec_fw_gpu_addr));
3543         }
3544         mutex_unlock(&adev->srbm_mutex);
3545         soc21_grbm_select(adev, 0, 0, 0, 0);
3546
3547         /* Trigger an invalidation of the L1 instruction caches */
3548         tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL);
3549         tmp = REG_SET_FIELD(tmp, CP_MEC_DC_OP_CNTL, INVALIDATE_DCACHE, 1);
3550         WREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL, tmp);
3551
3552         /* Wait for invalidation complete */
3553         for (i = 0; i < usec_timeout; i++) {
3554                 tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL);
3555                 if (1 == REG_GET_FIELD(tmp, CP_MEC_DC_OP_CNTL,
3556                                        INVALIDATE_DCACHE_COMPLETE))
3557                         break;
3558                 udelay(1);
3559         }
3560
3561         if (i >= usec_timeout) {
3562                 dev_err(adev->dev, "failed to invalidate instruction cache\n");
3563                 return -EINVAL;
3564         }
3565
3566         /* Trigger an invalidation of the L1 instruction caches */
3567         tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL);
3568         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_OP_CNTL, INVALIDATE_CACHE, 1);
3569         WREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL, tmp);
3570
3571         /* Wait for invalidation complete */
3572         for (i = 0; i < usec_timeout; i++) {
3573                 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL);
3574                 if (1 == REG_GET_FIELD(tmp, CP_CPC_IC_OP_CNTL,
3575                                        INVALIDATE_CACHE_COMPLETE))
3576                         break;
3577                 udelay(1);
3578         }
3579
3580         if (i >= usec_timeout) {
3581                 dev_err(adev->dev, "failed to invalidate instruction cache\n");
3582                 return -EINVAL;
3583         }
3584
3585         return 0;
3586 }
3587
3588 static void gfx_v11_0_kiq_setting(struct amdgpu_ring *ring)
3589 {
3590         uint32_t tmp;
3591         struct amdgpu_device *adev = ring->adev;
3592
3593         /* tell RLC which is KIQ queue */
3594         tmp = RREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS);
3595         tmp &= 0xffffff00;
3596         tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue);
3597         WREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS, tmp);
3598         tmp |= 0x80;
3599         WREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS, tmp);
3600 }
3601
3602 static void gfx_v11_0_cp_set_doorbell_range(struct amdgpu_device *adev)
3603 {
3604         /* set graphics engine doorbell range */
3605         WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_RANGE_LOWER,
3606                      (adev->doorbell_index.gfx_ring0 * 2) << 2);
3607         WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_RANGE_UPPER,
3608                      (adev->doorbell_index.gfx_userqueue_end * 2) << 2);
3609
3610         /* set compute engine doorbell range */
3611         WREG32_SOC15(GC, 0, regCP_MEC_DOORBELL_RANGE_LOWER,
3612                      (adev->doorbell_index.kiq * 2) << 2);
3613         WREG32_SOC15(GC, 0, regCP_MEC_DOORBELL_RANGE_UPPER,
3614                      (adev->doorbell_index.userqueue_end * 2) << 2);
3615 }
3616
3617 static int gfx_v11_0_gfx_mqd_init(struct amdgpu_device *adev, void *m,
3618                                   struct amdgpu_mqd_prop *prop)
3619 {
3620         struct v11_gfx_mqd *mqd = m;
3621         uint64_t hqd_gpu_addr, wb_gpu_addr;
3622         uint32_t tmp;
3623         uint32_t rb_bufsz;
3624
3625         /* set up gfx hqd wptr */
3626         mqd->cp_gfx_hqd_wptr = 0;
3627         mqd->cp_gfx_hqd_wptr_hi = 0;
3628
3629         /* set the pointer to the MQD */
3630         mqd->cp_mqd_base_addr = prop->mqd_gpu_addr & 0xfffffffc;
3631         mqd->cp_mqd_base_addr_hi = upper_32_bits(prop->mqd_gpu_addr);
3632
3633         /* set up mqd control */
3634         tmp = RREG32_SOC15(GC, 0, regCP_GFX_MQD_CONTROL);
3635         tmp = REG_SET_FIELD(tmp, CP_GFX_MQD_CONTROL, VMID, 0);
3636         tmp = REG_SET_FIELD(tmp, CP_GFX_MQD_CONTROL, PRIV_STATE, 1);
3637         tmp = REG_SET_FIELD(tmp, CP_GFX_MQD_CONTROL, CACHE_POLICY, 0);
3638         mqd->cp_gfx_mqd_control = tmp;
3639
3640         /* set up gfx_hqd_vimd with 0x0 to indicate the ring buffer's vmid */
3641         tmp = RREG32_SOC15(GC, 0, regCP_GFX_HQD_VMID);
3642         tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_VMID, VMID, 0);
3643         mqd->cp_gfx_hqd_vmid = 0;
3644
3645         /* set up default queue priority level
3646          * 0x0 = low priority, 0x1 = high priority */
3647         tmp = RREG32_SOC15(GC, 0, regCP_GFX_HQD_QUEUE_PRIORITY);
3648         tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_QUEUE_PRIORITY, PRIORITY_LEVEL, 0);
3649         mqd->cp_gfx_hqd_queue_priority = tmp;
3650
3651         /* set up time quantum */
3652         tmp = RREG32_SOC15(GC, 0, regCP_GFX_HQD_QUANTUM);
3653         tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_QUANTUM, QUANTUM_EN, 1);
3654         mqd->cp_gfx_hqd_quantum = tmp;
3655
3656         /* set up gfx hqd base. this is similar as CP_RB_BASE */
3657         hqd_gpu_addr = prop->hqd_base_gpu_addr >> 8;
3658         mqd->cp_gfx_hqd_base = hqd_gpu_addr;
3659         mqd->cp_gfx_hqd_base_hi = upper_32_bits(hqd_gpu_addr);
3660
3661         /* set up hqd_rptr_addr/_hi, similar as CP_RB_RPTR */
3662         wb_gpu_addr = prop->rptr_gpu_addr;
3663         mqd->cp_gfx_hqd_rptr_addr = wb_gpu_addr & 0xfffffffc;
3664         mqd->cp_gfx_hqd_rptr_addr_hi =
3665                 upper_32_bits(wb_gpu_addr) & 0xffff;
3666
3667         /* set up rb_wptr_poll addr */
3668         wb_gpu_addr = prop->wptr_gpu_addr;
3669         mqd->cp_rb_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc;
3670         mqd->cp_rb_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
3671
3672         /* set up the gfx_hqd_control, similar as CP_RB0_CNTL */
3673         rb_bufsz = order_base_2(prop->queue_size / 4) - 1;
3674         tmp = RREG32_SOC15(GC, 0, regCP_GFX_HQD_CNTL);
3675         tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, RB_BUFSZ, rb_bufsz);
3676         tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, RB_BLKSZ, rb_bufsz - 2);
3677 #ifdef __BIG_ENDIAN
3678         tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, BUF_SWAP, 1);
3679 #endif
3680         mqd->cp_gfx_hqd_cntl = tmp;
3681
3682         /* set up cp_doorbell_control */
3683         tmp = RREG32_SOC15(GC, 0, regCP_RB_DOORBELL_CONTROL);
3684         if (prop->use_doorbell) {
3685                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3686                                     DOORBELL_OFFSET, prop->doorbell_index);
3687                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3688                                     DOORBELL_EN, 1);
3689         } else
3690                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3691                                     DOORBELL_EN, 0);
3692         mqd->cp_rb_doorbell_control = tmp;
3693
3694         /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
3695         mqd->cp_gfx_hqd_rptr = RREG32_SOC15(GC, 0, regCP_GFX_HQD_RPTR);
3696
3697         /* active the queue */
3698         mqd->cp_gfx_hqd_active = 1;
3699
3700         return 0;
3701 }
3702
3703 static int gfx_v11_0_gfx_init_queue(struct amdgpu_ring *ring)
3704 {
3705         struct amdgpu_device *adev = ring->adev;
3706         struct v11_gfx_mqd *mqd = ring->mqd_ptr;
3707         int mqd_idx = ring - &adev->gfx.gfx_ring[0];
3708
3709         if (!amdgpu_in_reset(adev) && !adev->in_suspend) {
3710                 memset((void *)mqd, 0, sizeof(*mqd));
3711                 mutex_lock(&adev->srbm_mutex);
3712                 soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
3713                 amdgpu_ring_init_mqd(ring);
3714                 soc21_grbm_select(adev, 0, 0, 0, 0);
3715                 mutex_unlock(&adev->srbm_mutex);
3716                 if (adev->gfx.me.mqd_backup[mqd_idx])
3717                         memcpy(adev->gfx.me.mqd_backup[mqd_idx], mqd, sizeof(*mqd));
3718         } else {
3719                 /* restore mqd with the backup copy */
3720                 if (adev->gfx.me.mqd_backup[mqd_idx])
3721                         memcpy(mqd, adev->gfx.me.mqd_backup[mqd_idx], sizeof(*mqd));
3722                 /* reset the ring */
3723                 ring->wptr = 0;
3724                 *ring->wptr_cpu_addr = 0;
3725                 amdgpu_ring_clear_ring(ring);
3726         }
3727
3728         return 0;
3729 }
3730
3731 static int gfx_v11_0_cp_async_gfx_ring_resume(struct amdgpu_device *adev)
3732 {
3733         int r, i;
3734         struct amdgpu_ring *ring;
3735
3736         for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
3737                 ring = &adev->gfx.gfx_ring[i];
3738
3739                 r = amdgpu_bo_reserve(ring->mqd_obj, false);
3740                 if (unlikely(r != 0))
3741                         return r;
3742
3743                 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
3744                 if (!r) {
3745                         r = gfx_v11_0_gfx_init_queue(ring);
3746                         amdgpu_bo_kunmap(ring->mqd_obj);
3747                         ring->mqd_ptr = NULL;
3748                 }
3749                 amdgpu_bo_unreserve(ring->mqd_obj);
3750                 if (r)
3751                         return r;
3752         }
3753
3754         r = amdgpu_gfx_enable_kgq(adev, 0);
3755         if (r)
3756                 return r;
3757
3758         return gfx_v11_0_cp_gfx_start(adev);
3759 }
3760
3761 static int gfx_v11_0_compute_mqd_init(struct amdgpu_device *adev, void *m,
3762                                       struct amdgpu_mqd_prop *prop)
3763 {
3764         struct v11_compute_mqd *mqd = m;
3765         uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr;
3766         uint32_t tmp;
3767
3768         mqd->header = 0xC0310800;
3769         mqd->compute_pipelinestat_enable = 0x00000001;
3770         mqd->compute_static_thread_mgmt_se0 = 0xffffffff;
3771         mqd->compute_static_thread_mgmt_se1 = 0xffffffff;
3772         mqd->compute_static_thread_mgmt_se2 = 0xffffffff;
3773         mqd->compute_static_thread_mgmt_se3 = 0xffffffff;
3774         mqd->compute_misc_reserved = 0x00000007;
3775
3776         eop_base_addr = prop->eop_gpu_addr >> 8;
3777         mqd->cp_hqd_eop_base_addr_lo = eop_base_addr;
3778         mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr);
3779
3780         /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
3781         tmp = RREG32_SOC15(GC, 0, regCP_HQD_EOP_CONTROL);
3782         tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE,
3783                         (order_base_2(GFX11_MEC_HPD_SIZE / 4) - 1));
3784
3785         mqd->cp_hqd_eop_control = tmp;
3786
3787         /* enable doorbell? */
3788         tmp = RREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL);
3789
3790         if (prop->use_doorbell) {
3791                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3792                                     DOORBELL_OFFSET, prop->doorbell_index);
3793                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3794                                     DOORBELL_EN, 1);
3795                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3796                                     DOORBELL_SOURCE, 0);
3797                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3798                                     DOORBELL_HIT, 0);
3799         } else {
3800                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3801                                     DOORBELL_EN, 0);
3802         }
3803
3804         mqd->cp_hqd_pq_doorbell_control = tmp;
3805
3806         /* disable the queue if it's active */
3807         mqd->cp_hqd_dequeue_request = 0;
3808         mqd->cp_hqd_pq_rptr = 0;
3809         mqd->cp_hqd_pq_wptr_lo = 0;
3810         mqd->cp_hqd_pq_wptr_hi = 0;
3811
3812         /* set the pointer to the MQD */
3813         mqd->cp_mqd_base_addr_lo = prop->mqd_gpu_addr & 0xfffffffc;
3814         mqd->cp_mqd_base_addr_hi = upper_32_bits(prop->mqd_gpu_addr);
3815
3816         /* set MQD vmid to 0 */
3817         tmp = RREG32_SOC15(GC, 0, regCP_MQD_CONTROL);
3818         tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0);
3819         mqd->cp_mqd_control = tmp;
3820
3821         /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
3822         hqd_gpu_addr = prop->hqd_base_gpu_addr >> 8;
3823         mqd->cp_hqd_pq_base_lo = hqd_gpu_addr;
3824         mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
3825
3826         /* set up the HQD, this is similar to CP_RB0_CNTL */
3827         tmp = RREG32_SOC15(GC, 0, regCP_HQD_PQ_CONTROL);
3828         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE,
3829                             (order_base_2(prop->queue_size / 4) - 1));
3830         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE,
3831                             (order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1));
3832         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 0);
3833         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, TUNNEL_DISPATCH, 0);
3834         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1);
3835         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1);
3836         mqd->cp_hqd_pq_control = tmp;
3837
3838         /* set the wb address whether it's enabled or not */
3839         wb_gpu_addr = prop->rptr_gpu_addr;
3840         mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc;
3841         mqd->cp_hqd_pq_rptr_report_addr_hi =
3842                 upper_32_bits(wb_gpu_addr) & 0xffff;
3843
3844         /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
3845         wb_gpu_addr = prop->wptr_gpu_addr;
3846         mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc;
3847         mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
3848
3849         tmp = 0;
3850         /* enable the doorbell if requested */
3851         if (prop->use_doorbell) {
3852                 tmp = RREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL);
3853                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3854                                 DOORBELL_OFFSET, prop->doorbell_index);
3855
3856                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3857                                     DOORBELL_EN, 1);
3858                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3859                                     DOORBELL_SOURCE, 0);
3860                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3861                                     DOORBELL_HIT, 0);
3862         }
3863
3864         mqd->cp_hqd_pq_doorbell_control = tmp;
3865
3866         /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
3867         mqd->cp_hqd_pq_rptr = RREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR);
3868
3869         /* set the vmid for the queue */
3870         mqd->cp_hqd_vmid = 0;
3871
3872         tmp = RREG32_SOC15(GC, 0, regCP_HQD_PERSISTENT_STATE);
3873         tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x55);
3874         mqd->cp_hqd_persistent_state = tmp;
3875
3876         /* set MIN_IB_AVAIL_SIZE */
3877         tmp = RREG32_SOC15(GC, 0, regCP_HQD_IB_CONTROL);
3878         tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 3);
3879         mqd->cp_hqd_ib_control = tmp;
3880
3881         /* set static priority for a compute queue/ring */
3882         mqd->cp_hqd_pipe_priority = prop->hqd_pipe_priority;
3883         mqd->cp_hqd_queue_priority = prop->hqd_queue_priority;
3884
3885         mqd->cp_hqd_active = prop->hqd_active;
3886
3887         return 0;
3888 }
3889
3890 static int gfx_v11_0_kiq_init_register(struct amdgpu_ring *ring)
3891 {
3892         struct amdgpu_device *adev = ring->adev;
3893         struct v11_compute_mqd *mqd = ring->mqd_ptr;
3894         int j;
3895
3896         /* inactivate the queue */
3897         if (amdgpu_sriov_vf(adev))
3898                 WREG32_SOC15(GC, 0, regCP_HQD_ACTIVE, 0);
3899
3900         /* disable wptr polling */
3901         WREG32_FIELD15_PREREG(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0);
3902
3903         /* write the EOP addr */
3904         WREG32_SOC15(GC, 0, regCP_HQD_EOP_BASE_ADDR,
3905                mqd->cp_hqd_eop_base_addr_lo);
3906         WREG32_SOC15(GC, 0, regCP_HQD_EOP_BASE_ADDR_HI,
3907                mqd->cp_hqd_eop_base_addr_hi);
3908
3909         /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
3910         WREG32_SOC15(GC, 0, regCP_HQD_EOP_CONTROL,
3911                mqd->cp_hqd_eop_control);
3912
3913         /* enable doorbell? */
3914         WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL,
3915                mqd->cp_hqd_pq_doorbell_control);
3916
3917         /* disable the queue if it's active */
3918         if (RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) & 1) {
3919                 WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST, 1);
3920                 for (j = 0; j < adev->usec_timeout; j++) {
3921                         if (!(RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) & 1))
3922                                 break;
3923                         udelay(1);
3924                 }
3925                 WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST,
3926                        mqd->cp_hqd_dequeue_request);
3927                 WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR,
3928                        mqd->cp_hqd_pq_rptr);
3929                 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_LO,
3930                        mqd->cp_hqd_pq_wptr_lo);
3931                 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_HI,
3932                        mqd->cp_hqd_pq_wptr_hi);
3933         }
3934
3935         /* set the pointer to the MQD */
3936         WREG32_SOC15(GC, 0, regCP_MQD_BASE_ADDR,
3937                mqd->cp_mqd_base_addr_lo);
3938         WREG32_SOC15(GC, 0, regCP_MQD_BASE_ADDR_HI,
3939                mqd->cp_mqd_base_addr_hi);
3940
3941         /* set MQD vmid to 0 */
3942         WREG32_SOC15(GC, 0, regCP_MQD_CONTROL,
3943                mqd->cp_mqd_control);
3944
3945         /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
3946         WREG32_SOC15(GC, 0, regCP_HQD_PQ_BASE,
3947                mqd->cp_hqd_pq_base_lo);
3948         WREG32_SOC15(GC, 0, regCP_HQD_PQ_BASE_HI,
3949                mqd->cp_hqd_pq_base_hi);
3950
3951         /* set up the HQD, this is similar to CP_RB0_CNTL */
3952         WREG32_SOC15(GC, 0, regCP_HQD_PQ_CONTROL,
3953                mqd->cp_hqd_pq_control);
3954
3955         /* set the wb address whether it's enabled or not */
3956         WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR_REPORT_ADDR,
3957                 mqd->cp_hqd_pq_rptr_report_addr_lo);
3958         WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR_REPORT_ADDR_HI,
3959                 mqd->cp_hqd_pq_rptr_report_addr_hi);
3960
3961         /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
3962         WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR,
3963                mqd->cp_hqd_pq_wptr_poll_addr_lo);
3964         WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR_HI,
3965                mqd->cp_hqd_pq_wptr_poll_addr_hi);
3966
3967         /* enable the doorbell if requested */
3968         if (ring->use_doorbell) {
3969                 WREG32_SOC15(GC, 0, regCP_MEC_DOORBELL_RANGE_LOWER,
3970                         (adev->doorbell_index.kiq * 2) << 2);
3971                 WREG32_SOC15(GC, 0, regCP_MEC_DOORBELL_RANGE_UPPER,
3972                         (adev->doorbell_index.userqueue_end * 2) << 2);
3973         }
3974
3975         WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL,
3976                mqd->cp_hqd_pq_doorbell_control);
3977
3978         /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
3979         WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_LO,
3980                mqd->cp_hqd_pq_wptr_lo);
3981         WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_HI,
3982                mqd->cp_hqd_pq_wptr_hi);
3983
3984         /* set the vmid for the queue */
3985         WREG32_SOC15(GC, 0, regCP_HQD_VMID, mqd->cp_hqd_vmid);
3986
3987         WREG32_SOC15(GC, 0, regCP_HQD_PERSISTENT_STATE,
3988                mqd->cp_hqd_persistent_state);
3989
3990         /* activate the queue */
3991         WREG32_SOC15(GC, 0, regCP_HQD_ACTIVE,
3992                mqd->cp_hqd_active);
3993
3994         if (ring->use_doorbell)
3995                 WREG32_FIELD15_PREREG(GC, 0, CP_PQ_STATUS, DOORBELL_ENABLE, 1);
3996
3997         return 0;
3998 }
3999
4000 static int gfx_v11_0_kiq_init_queue(struct amdgpu_ring *ring)
4001 {
4002         struct amdgpu_device *adev = ring->adev;
4003         struct v11_compute_mqd *mqd = ring->mqd_ptr;
4004
4005         gfx_v11_0_kiq_setting(ring);
4006
4007         if (amdgpu_in_reset(adev)) { /* for GPU_RESET case */
4008                 /* reset MQD to a clean status */
4009                 if (adev->gfx.kiq[0].mqd_backup)
4010                         memcpy(mqd, adev->gfx.kiq[0].mqd_backup, sizeof(*mqd));
4011
4012                 /* reset ring buffer */
4013                 ring->wptr = 0;
4014                 amdgpu_ring_clear_ring(ring);
4015
4016                 mutex_lock(&adev->srbm_mutex);
4017                 soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
4018                 gfx_v11_0_kiq_init_register(ring);
4019                 soc21_grbm_select(adev, 0, 0, 0, 0);
4020                 mutex_unlock(&adev->srbm_mutex);
4021         } else {
4022                 memset((void *)mqd, 0, sizeof(*mqd));
4023                 if (amdgpu_sriov_vf(adev) && adev->in_suspend)
4024                         amdgpu_ring_clear_ring(ring);
4025                 mutex_lock(&adev->srbm_mutex);
4026                 soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
4027                 amdgpu_ring_init_mqd(ring);
4028                 gfx_v11_0_kiq_init_register(ring);
4029                 soc21_grbm_select(adev, 0, 0, 0, 0);
4030                 mutex_unlock(&adev->srbm_mutex);
4031
4032                 if (adev->gfx.kiq[0].mqd_backup)
4033                         memcpy(adev->gfx.kiq[0].mqd_backup, mqd, sizeof(*mqd));
4034         }
4035
4036         return 0;
4037 }
4038
4039 static int gfx_v11_0_kcq_init_queue(struct amdgpu_ring *ring)
4040 {
4041         struct amdgpu_device *adev = ring->adev;
4042         struct v11_compute_mqd *mqd = ring->mqd_ptr;
4043         int mqd_idx = ring - &adev->gfx.compute_ring[0];
4044
4045         if (!amdgpu_in_reset(adev) && !adev->in_suspend) {
4046                 memset((void *)mqd, 0, sizeof(*mqd));
4047                 mutex_lock(&adev->srbm_mutex);
4048                 soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
4049                 amdgpu_ring_init_mqd(ring);
4050                 soc21_grbm_select(adev, 0, 0, 0, 0);
4051                 mutex_unlock(&adev->srbm_mutex);
4052
4053                 if (adev->gfx.mec.mqd_backup[mqd_idx])
4054                         memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(*mqd));
4055         } else {
4056                 /* restore MQD to a clean status */
4057                 if (adev->gfx.mec.mqd_backup[mqd_idx])
4058                         memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(*mqd));
4059                 /* reset ring buffer */
4060                 ring->wptr = 0;
4061                 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, 0);
4062                 amdgpu_ring_clear_ring(ring);
4063         }
4064
4065         return 0;
4066 }
4067
4068 static int gfx_v11_0_kiq_resume(struct amdgpu_device *adev)
4069 {
4070         struct amdgpu_ring *ring;
4071         int r;
4072
4073         ring = &adev->gfx.kiq[0].ring;
4074
4075         r = amdgpu_bo_reserve(ring->mqd_obj, false);
4076         if (unlikely(r != 0))
4077                 return r;
4078
4079         r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
4080         if (unlikely(r != 0)) {
4081                 amdgpu_bo_unreserve(ring->mqd_obj);
4082                 return r;
4083         }
4084
4085         gfx_v11_0_kiq_init_queue(ring);
4086         amdgpu_bo_kunmap(ring->mqd_obj);
4087         ring->mqd_ptr = NULL;
4088         amdgpu_bo_unreserve(ring->mqd_obj);
4089         ring->sched.ready = true;
4090         return 0;
4091 }
4092
4093 static int gfx_v11_0_kcq_resume(struct amdgpu_device *adev)
4094 {
4095         struct amdgpu_ring *ring = NULL;
4096         int r = 0, i;
4097
4098         if (!amdgpu_async_gfx_ring)
4099                 gfx_v11_0_cp_compute_enable(adev, true);
4100
4101         for (i = 0; i < adev->gfx.num_compute_rings; i++) {
4102                 ring = &adev->gfx.compute_ring[i];
4103
4104                 r = amdgpu_bo_reserve(ring->mqd_obj, false);
4105                 if (unlikely(r != 0))
4106                         goto done;
4107                 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
4108                 if (!r) {
4109                         r = gfx_v11_0_kcq_init_queue(ring);
4110                         amdgpu_bo_kunmap(ring->mqd_obj);
4111                         ring->mqd_ptr = NULL;
4112                 }
4113                 amdgpu_bo_unreserve(ring->mqd_obj);
4114                 if (r)
4115                         goto done;
4116         }
4117
4118         r = amdgpu_gfx_enable_kcq(adev, 0);
4119 done:
4120         return r;
4121 }
4122
4123 static int gfx_v11_0_cp_resume(struct amdgpu_device *adev)
4124 {
4125         int r, i;
4126         struct amdgpu_ring *ring;
4127
4128         if (!(adev->flags & AMD_IS_APU))
4129                 gfx_v11_0_enable_gui_idle_interrupt(adev, false);
4130
4131         if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
4132                 /* legacy firmware loading */
4133                 r = gfx_v11_0_cp_gfx_load_microcode(adev);
4134                 if (r)
4135                         return r;
4136
4137                 if (adev->gfx.rs64_enable)
4138                         r = gfx_v11_0_cp_compute_load_microcode_rs64(adev);
4139                 else
4140                         r = gfx_v11_0_cp_compute_load_microcode(adev);
4141                 if (r)
4142                         return r;
4143         }
4144
4145         gfx_v11_0_cp_set_doorbell_range(adev);
4146
4147         if (amdgpu_async_gfx_ring) {
4148                 gfx_v11_0_cp_compute_enable(adev, true);
4149                 gfx_v11_0_cp_gfx_enable(adev, true);
4150         }
4151
4152         if (adev->enable_mes_kiq && adev->mes.kiq_hw_init)
4153                 r = amdgpu_mes_kiq_hw_init(adev);
4154         else
4155                 r = gfx_v11_0_kiq_resume(adev);
4156         if (r)
4157                 return r;
4158
4159         r = gfx_v11_0_kcq_resume(adev);
4160         if (r)
4161                 return r;
4162
4163         if (!amdgpu_async_gfx_ring) {
4164                 r = gfx_v11_0_cp_gfx_resume(adev);
4165                 if (r)
4166                         return r;
4167         } else {
4168                 r = gfx_v11_0_cp_async_gfx_ring_resume(adev);
4169                 if (r)
4170                         return r;
4171         }
4172
4173         for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
4174                 ring = &adev->gfx.gfx_ring[i];
4175                 r = amdgpu_ring_test_helper(ring);
4176                 if (r)
4177                         return r;
4178         }
4179
4180         for (i = 0; i < adev->gfx.num_compute_rings; i++) {
4181                 ring = &adev->gfx.compute_ring[i];
4182                 r = amdgpu_ring_test_helper(ring);
4183                 if (r)
4184                         return r;
4185         }
4186
4187         return 0;
4188 }
4189
4190 static void gfx_v11_0_cp_enable(struct amdgpu_device *adev, bool enable)
4191 {
4192         gfx_v11_0_cp_gfx_enable(adev, enable);
4193         gfx_v11_0_cp_compute_enable(adev, enable);
4194 }
4195
4196 static int gfx_v11_0_gfxhub_enable(struct amdgpu_device *adev)
4197 {
4198         int r;
4199         bool value;
4200
4201         r = adev->gfxhub.funcs->gart_enable(adev);
4202         if (r)
4203                 return r;
4204
4205         adev->hdp.funcs->flush_hdp(adev, NULL);
4206
4207         value = (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS) ?
4208                 false : true;
4209
4210         adev->gfxhub.funcs->set_fault_enable_default(adev, value);
4211         amdgpu_gmc_flush_gpu_tlb(adev, 0, AMDGPU_GFXHUB(0), 0);
4212
4213         return 0;
4214 }
4215
4216 static void gfx_v11_0_select_cp_fw_arch(struct amdgpu_device *adev)
4217 {
4218         u32 tmp;
4219
4220         /* select RS64 */
4221         if (adev->gfx.rs64_enable) {
4222                 tmp = RREG32_SOC15(GC, 0, regCP_GFX_CNTL);
4223                 tmp = REG_SET_FIELD(tmp, CP_GFX_CNTL, ENGINE_SEL, 1);
4224                 WREG32_SOC15(GC, 0, regCP_GFX_CNTL, tmp);
4225
4226                 tmp = RREG32_SOC15(GC, 0, regCP_MEC_ISA_CNTL);
4227                 tmp = REG_SET_FIELD(tmp, CP_MEC_ISA_CNTL, ISA_MODE, 1);
4228                 WREG32_SOC15(GC, 0, regCP_MEC_ISA_CNTL, tmp);
4229         }
4230
4231         if (amdgpu_emu_mode == 1)
4232                 msleep(100);
4233 }
4234
4235 static int get_gb_addr_config(struct amdgpu_device * adev)
4236 {
4237         u32 gb_addr_config;
4238
4239         gb_addr_config = RREG32_SOC15(GC, 0, regGB_ADDR_CONFIG);
4240         if (gb_addr_config == 0)
4241                 return -EINVAL;
4242
4243         adev->gfx.config.gb_addr_config_fields.num_pkrs =
4244                 1 << REG_GET_FIELD(gb_addr_config, GB_ADDR_CONFIG, NUM_PKRS);
4245
4246         adev->gfx.config.gb_addr_config = gb_addr_config;
4247
4248         adev->gfx.config.gb_addr_config_fields.num_pipes = 1 <<
4249                         REG_GET_FIELD(adev->gfx.config.gb_addr_config,
4250                                       GB_ADDR_CONFIG, NUM_PIPES);
4251
4252         adev->gfx.config.max_tile_pipes =
4253                 adev->gfx.config.gb_addr_config_fields.num_pipes;
4254
4255         adev->gfx.config.gb_addr_config_fields.max_compress_frags = 1 <<
4256                         REG_GET_FIELD(adev->gfx.config.gb_addr_config,
4257                                       GB_ADDR_CONFIG, MAX_COMPRESSED_FRAGS);
4258         adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 1 <<
4259                         REG_GET_FIELD(adev->gfx.config.gb_addr_config,
4260                                       GB_ADDR_CONFIG, NUM_RB_PER_SE);
4261         adev->gfx.config.gb_addr_config_fields.num_se = 1 <<
4262                         REG_GET_FIELD(adev->gfx.config.gb_addr_config,
4263                                       GB_ADDR_CONFIG, NUM_SHADER_ENGINES);
4264         adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 1 << (8 +
4265                         REG_GET_FIELD(adev->gfx.config.gb_addr_config,
4266                                       GB_ADDR_CONFIG, PIPE_INTERLEAVE_SIZE));
4267
4268         return 0;
4269 }
4270
4271 static void gfx_v11_0_disable_gpa_mode(struct amdgpu_device *adev)
4272 {
4273         uint32_t data;
4274
4275         data = RREG32_SOC15(GC, 0, regCPC_PSP_DEBUG);
4276         data |= CPC_PSP_DEBUG__GPA_OVERRIDE_MASK;
4277         WREG32_SOC15(GC, 0, regCPC_PSP_DEBUG, data);
4278
4279         data = RREG32_SOC15(GC, 0, regCPG_PSP_DEBUG);
4280         data |= CPG_PSP_DEBUG__GPA_OVERRIDE_MASK;
4281         WREG32_SOC15(GC, 0, regCPG_PSP_DEBUG, data);
4282 }
4283
4284 static int gfx_v11_0_hw_init(void *handle)
4285 {
4286         int r;
4287         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4288
4289         if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) {
4290                 if (adev->gfx.imu.funcs) {
4291                         /* RLC autoload sequence 1: Program rlc ram */
4292                         if (adev->gfx.imu.funcs->program_rlc_ram)
4293                                 adev->gfx.imu.funcs->program_rlc_ram(adev);
4294                 }
4295                 /* rlc autoload firmware */
4296                 r = gfx_v11_0_rlc_backdoor_autoload_enable(adev);
4297                 if (r)
4298                         return r;
4299         } else {
4300                 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
4301                         if (adev->gfx.imu.funcs && (amdgpu_dpm > 0)) {
4302                                 if (adev->gfx.imu.funcs->load_microcode)
4303                                         adev->gfx.imu.funcs->load_microcode(adev);
4304                                 if (adev->gfx.imu.funcs->setup_imu)
4305                                         adev->gfx.imu.funcs->setup_imu(adev);
4306                                 if (adev->gfx.imu.funcs->start_imu)
4307                                         adev->gfx.imu.funcs->start_imu(adev);
4308                         }
4309
4310                         /* disable gpa mode in backdoor loading */
4311                         gfx_v11_0_disable_gpa_mode(adev);
4312                 }
4313         }
4314
4315         if ((adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) ||
4316             (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP)) {
4317                 r = gfx_v11_0_wait_for_rlc_autoload_complete(adev);
4318                 if (r) {
4319                         dev_err(adev->dev, "(%d) failed to wait rlc autoload complete\n", r);
4320                         return r;
4321                 }
4322         }
4323
4324         adev->gfx.is_poweron = true;
4325
4326         if(get_gb_addr_config(adev))
4327                 DRM_WARN("Invalid gb_addr_config !\n");
4328
4329         if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP &&
4330             adev->gfx.rs64_enable)
4331                 gfx_v11_0_config_gfx_rs64(adev);
4332
4333         r = gfx_v11_0_gfxhub_enable(adev);
4334         if (r)
4335                 return r;
4336
4337         if (!amdgpu_emu_mode)
4338                 gfx_v11_0_init_golden_registers(adev);
4339
4340         if ((adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) ||
4341             (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO && amdgpu_dpm == 1)) {
4342                 /**
4343                  * For gfx 11, rlc firmware loading relies on smu firmware is
4344                  * loaded firstly, so in direct type, it has to load smc ucode
4345                  * here before rlc.
4346                  */
4347                 if (!(adev->flags & AMD_IS_APU)) {
4348                         r = amdgpu_pm_load_smu_firmware(adev, NULL);
4349                         if (r)
4350                                 return r;
4351                 }
4352         }
4353
4354         gfx_v11_0_constants_init(adev);
4355
4356         if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP)
4357                 gfx_v11_0_select_cp_fw_arch(adev);
4358
4359         if (adev->nbio.funcs->gc_doorbell_init)
4360                 adev->nbio.funcs->gc_doorbell_init(adev);
4361
4362         r = gfx_v11_0_rlc_resume(adev);
4363         if (r)
4364                 return r;
4365
4366         /*
4367          * init golden registers and rlc resume may override some registers,
4368          * reconfig them here
4369          */
4370         gfx_v11_0_tcp_harvest(adev);
4371
4372         r = gfx_v11_0_cp_resume(adev);
4373         if (r)
4374                 return r;
4375
4376         return r;
4377 }
4378
4379 static int gfx_v11_0_hw_fini(void *handle)
4380 {
4381         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4382
4383         amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
4384         amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
4385
4386         if (!adev->no_hw_access) {
4387                 if (amdgpu_async_gfx_ring) {
4388                         if (amdgpu_gfx_disable_kgq(adev, 0))
4389                                 DRM_ERROR("KGQ disable failed\n");
4390                 }
4391
4392                 if (amdgpu_gfx_disable_kcq(adev, 0))
4393                         DRM_ERROR("KCQ disable failed\n");
4394
4395                 amdgpu_mes_kiq_hw_fini(adev);
4396         }
4397
4398         if (amdgpu_sriov_vf(adev))
4399                 /* Remove the steps disabling CPG and clearing KIQ position,
4400                  * so that CP could perform IDLE-SAVE during switch. Those
4401                  * steps are necessary to avoid a DMAR error in gfx9 but it is
4402                  * not reproduced on gfx11.
4403                  */
4404                 return 0;
4405
4406         gfx_v11_0_cp_enable(adev, false);
4407         gfx_v11_0_enable_gui_idle_interrupt(adev, false);
4408
4409         adev->gfxhub.funcs->gart_disable(adev);
4410
4411         adev->gfx.is_poweron = false;
4412
4413         return 0;
4414 }
4415
4416 static int gfx_v11_0_suspend(void *handle)
4417 {
4418         return gfx_v11_0_hw_fini(handle);
4419 }
4420
4421 static int gfx_v11_0_resume(void *handle)
4422 {
4423         return gfx_v11_0_hw_init(handle);
4424 }
4425
4426 static bool gfx_v11_0_is_idle(void *handle)
4427 {
4428         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4429
4430         if (REG_GET_FIELD(RREG32_SOC15(GC, 0, regGRBM_STATUS),
4431                                 GRBM_STATUS, GUI_ACTIVE))
4432                 return false;
4433         else
4434                 return true;
4435 }
4436
4437 static int gfx_v11_0_wait_for_idle(void *handle)
4438 {
4439         unsigned i;
4440         u32 tmp;
4441         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4442
4443         for (i = 0; i < adev->usec_timeout; i++) {
4444                 /* read MC_STATUS */
4445                 tmp = RREG32_SOC15(GC, 0, regGRBM_STATUS) &
4446                         GRBM_STATUS__GUI_ACTIVE_MASK;
4447
4448                 if (!REG_GET_FIELD(tmp, GRBM_STATUS, GUI_ACTIVE))
4449                         return 0;
4450                 udelay(1);
4451         }
4452         return -ETIMEDOUT;
4453 }
4454
4455 static int gfx_v11_0_soft_reset(void *handle)
4456 {
4457         u32 grbm_soft_reset = 0;
4458         u32 tmp;
4459         int i, j, k;
4460         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4461
4462         tmp = RREG32_SOC15(GC, 0, regCP_INT_CNTL);
4463         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CMP_BUSY_INT_ENABLE, 0);
4464         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CNTX_BUSY_INT_ENABLE, 0);
4465         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CNTX_EMPTY_INT_ENABLE, 0);
4466         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, GFX_IDLE_INT_ENABLE, 0);
4467         WREG32_SOC15(GC, 0, regCP_INT_CNTL, tmp);
4468
4469         gfx_v11_0_set_safe_mode(adev, 0);
4470
4471         for (i = 0; i < adev->gfx.mec.num_mec; ++i) {
4472                 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {
4473                         for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {
4474                                 tmp = RREG32_SOC15(GC, 0, regGRBM_GFX_CNTL);
4475                                 tmp = REG_SET_FIELD(tmp, GRBM_GFX_CNTL, MEID, i);
4476                                 tmp = REG_SET_FIELD(tmp, GRBM_GFX_CNTL, QUEUEID, j);
4477                                 tmp = REG_SET_FIELD(tmp, GRBM_GFX_CNTL, PIPEID, k);
4478                                 WREG32_SOC15(GC, 0, regGRBM_GFX_CNTL, tmp);
4479
4480                                 WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST, 0x2);
4481                                 WREG32_SOC15(GC, 0, regSPI_COMPUTE_QUEUE_RESET, 0x1);
4482                         }
4483                 }
4484         }
4485         for (i = 0; i < adev->gfx.me.num_me; ++i) {
4486                 for (j = 0; j < adev->gfx.me.num_queue_per_pipe; j++) {
4487                         for (k = 0; k < adev->gfx.me.num_pipe_per_me; k++) {
4488                                 tmp = RREG32_SOC15(GC, 0, regGRBM_GFX_CNTL);
4489                                 tmp = REG_SET_FIELD(tmp, GRBM_GFX_CNTL, MEID, i);
4490                                 tmp = REG_SET_FIELD(tmp, GRBM_GFX_CNTL, QUEUEID, j);
4491                                 tmp = REG_SET_FIELD(tmp, GRBM_GFX_CNTL, PIPEID, k);
4492                                 WREG32_SOC15(GC, 0, regGRBM_GFX_CNTL, tmp);
4493
4494                                 WREG32_SOC15(GC, 0, regCP_GFX_HQD_DEQUEUE_REQUEST, 0x1);
4495                         }
4496                 }
4497         }
4498
4499         WREG32_SOC15(GC, 0, regCP_VMID_RESET, 0xfffffffe);
4500
4501         // Read CP_VMID_RESET register three times.
4502         // to get sufficient time for GFX_HQD_ACTIVE reach 0
4503         RREG32_SOC15(GC, 0, regCP_VMID_RESET);
4504         RREG32_SOC15(GC, 0, regCP_VMID_RESET);
4505         RREG32_SOC15(GC, 0, regCP_VMID_RESET);
4506
4507         for (i = 0; i < adev->usec_timeout; i++) {
4508                 if (!RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) &&
4509                     !RREG32_SOC15(GC, 0, regCP_GFX_HQD_ACTIVE))
4510                         break;
4511                 udelay(1);
4512         }
4513         if (i >= adev->usec_timeout) {
4514                 printk("Failed to wait all pipes clean\n");
4515                 return -EINVAL;
4516         }
4517
4518         /**********  trigger soft reset  ***********/
4519         grbm_soft_reset = RREG32_SOC15(GC, 0, regGRBM_SOFT_RESET);
4520         grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
4521                                         SOFT_RESET_CP, 1);
4522         grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
4523                                         SOFT_RESET_GFX, 1);
4524         grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
4525                                         SOFT_RESET_CPF, 1);
4526         grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
4527                                         SOFT_RESET_CPC, 1);
4528         grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
4529                                         SOFT_RESET_CPG, 1);
4530         WREG32_SOC15(GC, 0, regGRBM_SOFT_RESET, grbm_soft_reset);
4531         /**********  exit soft reset  ***********/
4532         grbm_soft_reset = RREG32_SOC15(GC, 0, regGRBM_SOFT_RESET);
4533         grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
4534                                         SOFT_RESET_CP, 0);
4535         grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
4536                                         SOFT_RESET_GFX, 0);
4537         grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
4538                                         SOFT_RESET_CPF, 0);
4539         grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
4540                                         SOFT_RESET_CPC, 0);
4541         grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
4542                                         SOFT_RESET_CPG, 0);
4543         WREG32_SOC15(GC, 0, regGRBM_SOFT_RESET, grbm_soft_reset);
4544
4545         tmp = RREG32_SOC15(GC, 0, regCP_SOFT_RESET_CNTL);
4546         tmp = REG_SET_FIELD(tmp, CP_SOFT_RESET_CNTL, CMP_HQD_REG_RESET, 0x1);
4547         WREG32_SOC15(GC, 0, regCP_SOFT_RESET_CNTL, tmp);
4548
4549         WREG32_SOC15(GC, 0, regCP_ME_CNTL, 0x0);
4550         WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, 0x0);
4551
4552         for (i = 0; i < adev->usec_timeout; i++) {
4553                 if (!RREG32_SOC15(GC, 0, regCP_VMID_RESET))
4554                         break;
4555                 udelay(1);
4556         }
4557         if (i >= adev->usec_timeout) {
4558                 printk("Failed to wait CP_VMID_RESET to 0\n");
4559                 return -EINVAL;
4560         }
4561
4562         tmp = RREG32_SOC15(GC, 0, regCP_INT_CNTL);
4563         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CMP_BUSY_INT_ENABLE, 1);
4564         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CNTX_BUSY_INT_ENABLE, 1);
4565         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CNTX_EMPTY_INT_ENABLE, 1);
4566         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, GFX_IDLE_INT_ENABLE, 1);
4567         WREG32_SOC15(GC, 0, regCP_INT_CNTL, tmp);
4568
4569         gfx_v11_0_unset_safe_mode(adev, 0);
4570
4571         return gfx_v11_0_cp_resume(adev);
4572 }
4573
4574 static bool gfx_v11_0_check_soft_reset(void *handle)
4575 {
4576         int i, r;
4577         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4578         struct amdgpu_ring *ring;
4579         long tmo = msecs_to_jiffies(1000);
4580
4581         for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
4582                 ring = &adev->gfx.gfx_ring[i];
4583                 r = amdgpu_ring_test_ib(ring, tmo);
4584                 if (r)
4585                         return true;
4586         }
4587
4588         for (i = 0; i < adev->gfx.num_compute_rings; i++) {
4589                 ring = &adev->gfx.compute_ring[i];
4590                 r = amdgpu_ring_test_ib(ring, tmo);
4591                 if (r)
4592                         return true;
4593         }
4594
4595         return false;
4596 }
4597
4598 static int gfx_v11_0_post_soft_reset(void *handle)
4599 {
4600         /**
4601          * GFX soft reset will impact MES, need resume MES when do GFX soft reset
4602          */
4603         return amdgpu_mes_resume((struct amdgpu_device *)handle);
4604 }
4605
4606 static uint64_t gfx_v11_0_get_gpu_clock_counter(struct amdgpu_device *adev)
4607 {
4608         uint64_t clock;
4609         uint64_t clock_counter_lo, clock_counter_hi_pre, clock_counter_hi_after;
4610
4611         if (amdgpu_sriov_vf(adev)) {
4612                 amdgpu_gfx_off_ctrl(adev, false);
4613                 mutex_lock(&adev->gfx.gpu_clock_mutex);
4614                 clock_counter_hi_pre = (uint64_t)RREG32_SOC15(GC, 0, regCP_MES_MTIME_HI);
4615                 clock_counter_lo = (uint64_t)RREG32_SOC15(GC, 0, regCP_MES_MTIME_LO);
4616                 clock_counter_hi_after = (uint64_t)RREG32_SOC15(GC, 0, regCP_MES_MTIME_HI);
4617                 if (clock_counter_hi_pre != clock_counter_hi_after)
4618                         clock_counter_lo = (uint64_t)RREG32_SOC15(GC, 0, regCP_MES_MTIME_LO);
4619                 mutex_unlock(&adev->gfx.gpu_clock_mutex);
4620                 amdgpu_gfx_off_ctrl(adev, true);
4621         } else {
4622                 preempt_disable();
4623                 clock_counter_hi_pre = (uint64_t)RREG32_SOC15(SMUIO, 0, regGOLDEN_TSC_COUNT_UPPER);
4624                 clock_counter_lo = (uint64_t)RREG32_SOC15(SMUIO, 0, regGOLDEN_TSC_COUNT_LOWER);
4625                 clock_counter_hi_after = (uint64_t)RREG32_SOC15(SMUIO, 0, regGOLDEN_TSC_COUNT_UPPER);
4626                 if (clock_counter_hi_pre != clock_counter_hi_after)
4627                         clock_counter_lo = (uint64_t)RREG32_SOC15(SMUIO, 0, regGOLDEN_TSC_COUNT_LOWER);
4628                 preempt_enable();
4629         }
4630         clock = clock_counter_lo | (clock_counter_hi_after << 32ULL);
4631
4632         return clock;
4633 }
4634
4635 static void gfx_v11_0_ring_emit_gds_switch(struct amdgpu_ring *ring,
4636                                            uint32_t vmid,
4637                                            uint32_t gds_base, uint32_t gds_size,
4638                                            uint32_t gws_base, uint32_t gws_size,
4639                                            uint32_t oa_base, uint32_t oa_size)
4640 {
4641         struct amdgpu_device *adev = ring->adev;
4642
4643         /* GDS Base */
4644         gfx_v11_0_write_data_to_reg(ring, 0, false,
4645                                     SOC15_REG_OFFSET(GC, 0, regGDS_VMID0_BASE) + 2 * vmid,
4646                                     gds_base);
4647
4648         /* GDS Size */
4649         gfx_v11_0_write_data_to_reg(ring, 0, false,
4650                                     SOC15_REG_OFFSET(GC, 0, regGDS_VMID0_SIZE) + 2 * vmid,
4651                                     gds_size);
4652
4653         /* GWS */
4654         gfx_v11_0_write_data_to_reg(ring, 0, false,
4655                                     SOC15_REG_OFFSET(GC, 0, regGDS_GWS_VMID0) + vmid,
4656                                     gws_size << GDS_GWS_VMID0__SIZE__SHIFT | gws_base);
4657
4658         /* OA */
4659         gfx_v11_0_write_data_to_reg(ring, 0, false,
4660                                     SOC15_REG_OFFSET(GC, 0, regGDS_OA_VMID0) + vmid,
4661                                     (1 << (oa_size + oa_base)) - (1 << oa_base));
4662 }
4663
4664 static int gfx_v11_0_early_init(void *handle)
4665 {
4666         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4667
4668         adev->gfx.funcs = &gfx_v11_0_gfx_funcs;
4669
4670         adev->gfx.num_gfx_rings = GFX11_NUM_GFX_RINGS;
4671         adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev),
4672                                           AMDGPU_MAX_COMPUTE_RINGS);
4673
4674         gfx_v11_0_set_kiq_pm4_funcs(adev);
4675         gfx_v11_0_set_ring_funcs(adev);
4676         gfx_v11_0_set_irq_funcs(adev);
4677         gfx_v11_0_set_gds_init(adev);
4678         gfx_v11_0_set_rlc_funcs(adev);
4679         gfx_v11_0_set_mqd_funcs(adev);
4680         gfx_v11_0_set_imu_funcs(adev);
4681
4682         gfx_v11_0_init_rlcg_reg_access_ctrl(adev);
4683
4684         return gfx_v11_0_init_microcode(adev);
4685 }
4686
4687 static int gfx_v11_0_late_init(void *handle)
4688 {
4689         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4690         int r;
4691
4692         r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0);
4693         if (r)
4694                 return r;
4695
4696         r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0);
4697         if (r)
4698                 return r;
4699
4700         return 0;
4701 }
4702
4703 static bool gfx_v11_0_is_rlc_enabled(struct amdgpu_device *adev)
4704 {
4705         uint32_t rlc_cntl;
4706
4707         /* if RLC is not enabled, do nothing */
4708         rlc_cntl = RREG32_SOC15(GC, 0, regRLC_CNTL);
4709         return (REG_GET_FIELD(rlc_cntl, RLC_CNTL, RLC_ENABLE_F32)) ? true : false;
4710 }
4711
4712 static void gfx_v11_0_set_safe_mode(struct amdgpu_device *adev, int xcc_id)
4713 {
4714         uint32_t data;
4715         unsigned i;
4716
4717         data = RLC_SAFE_MODE__CMD_MASK;
4718         data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT);
4719
4720         WREG32_SOC15(GC, 0, regRLC_SAFE_MODE, data);
4721
4722         /* wait for RLC_SAFE_MODE */
4723         for (i = 0; i < adev->usec_timeout; i++) {
4724                 if (!REG_GET_FIELD(RREG32_SOC15(GC, 0, regRLC_SAFE_MODE),
4725                                    RLC_SAFE_MODE, CMD))
4726                         break;
4727                 udelay(1);
4728         }
4729 }
4730
4731 static void gfx_v11_0_unset_safe_mode(struct amdgpu_device *adev, int xcc_id)
4732 {
4733         WREG32_SOC15(GC, 0, regRLC_SAFE_MODE, RLC_SAFE_MODE__CMD_MASK);
4734 }
4735
4736 static void gfx_v11_0_update_perf_clk(struct amdgpu_device *adev,
4737                                       bool enable)
4738 {
4739         uint32_t def, data;
4740
4741         if (!(adev->cg_flags & AMD_CG_SUPPORT_GFX_PERF_CLK))
4742                 return;
4743
4744         def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
4745
4746         if (enable)
4747                 data &= ~RLC_CGTT_MGCG_OVERRIDE__PERFMON_CLOCK_STATE_MASK;
4748         else
4749                 data |= RLC_CGTT_MGCG_OVERRIDE__PERFMON_CLOCK_STATE_MASK;
4750
4751         if (def != data)
4752                 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data);
4753 }
4754
4755 static void gfx_v11_0_update_sram_fgcg(struct amdgpu_device *adev,
4756                                        bool enable)
4757 {
4758         uint32_t def, data;
4759
4760         if (!(adev->cg_flags & AMD_CG_SUPPORT_GFX_FGCG))
4761                 return;
4762
4763         def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
4764
4765         if (enable)
4766                 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_FGCG_OVERRIDE_MASK;
4767         else
4768                 data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_FGCG_OVERRIDE_MASK;
4769
4770         if (def != data)
4771                 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data);
4772 }
4773
4774 static void gfx_v11_0_update_repeater_fgcg(struct amdgpu_device *adev,
4775                                            bool enable)
4776 {
4777         uint32_t def, data;
4778
4779         if (!(adev->cg_flags & AMD_CG_SUPPORT_REPEATER_FGCG))
4780                 return;
4781
4782         def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
4783
4784         if (enable)
4785                 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_REPEATER_FGCG_OVERRIDE_MASK;
4786         else
4787                 data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_REPEATER_FGCG_OVERRIDE_MASK;
4788
4789         if (def != data)
4790                 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data);
4791 }
4792
4793 static void gfx_v11_0_update_medium_grain_clock_gating(struct amdgpu_device *adev,
4794                                                        bool enable)
4795 {
4796         uint32_t data, def;
4797
4798         if (!(adev->cg_flags & (AMD_CG_SUPPORT_GFX_MGCG | AMD_CG_SUPPORT_GFX_MGLS)))
4799                 return;
4800
4801         /* It is disabled by HW by default */
4802         if (enable) {
4803                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG) {
4804                         /* 1 - RLC_CGTT_MGCG_OVERRIDE */
4805                         def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
4806
4807                         data &= ~(RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
4808                                   RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK |
4809                                   RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK);
4810
4811                         if (def != data)
4812                                 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data);
4813                 }
4814         } else {
4815                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG) {
4816                         def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
4817
4818                         data |= (RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK |
4819                                  RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
4820                                  RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK);
4821
4822                         if (def != data)
4823                                 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data);
4824                 }
4825         }
4826 }
4827
4828 static void gfx_v11_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev,
4829                                                        bool enable)
4830 {
4831         uint32_t def, data;
4832
4833         if (!(adev->cg_flags &
4834               (AMD_CG_SUPPORT_GFX_CGCG |
4835               AMD_CG_SUPPORT_GFX_CGLS |
4836               AMD_CG_SUPPORT_GFX_3D_CGCG |
4837               AMD_CG_SUPPORT_GFX_3D_CGLS)))
4838                 return;
4839
4840         if (enable) {
4841                 def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
4842
4843                 /* unset CGCG override */
4844                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)
4845                         data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGCG_OVERRIDE_MASK;
4846                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
4847                         data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK;
4848                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG ||
4849                     adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS)
4850                         data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_GFX3D_CG_OVERRIDE_MASK;
4851
4852                 /* update CGCG override bits */
4853                 if (def != data)
4854                         WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data);
4855
4856                 /* enable cgcg FSM(0x0000363F) */
4857                 def = data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL);
4858
4859                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG) {
4860                         data &= ~RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD_MASK;
4861                         data |= (0x36 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
4862                                  RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
4863                 }
4864
4865                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) {
4866                         data &= ~RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY_MASK;
4867                         data |= (0x000F << RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
4868                                  RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;
4869                 }
4870
4871                 if (def != data)
4872                         WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL, data);
4873
4874                 /* Program RLC_CGCG_CGLS_CTRL_3D */
4875                 def = data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D);
4876
4877                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG) {
4878                         data &= ~RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD_MASK;
4879                         data |= (0x36 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
4880                                  RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK;
4881                 }
4882
4883                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS) {
4884                         data &= ~RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY_MASK;
4885                         data |= (0xf << RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
4886                                  RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK;
4887                 }
4888
4889                 if (def != data)
4890                         WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D, data);
4891
4892                 /* set IDLE_POLL_COUNT(0x00900100) */
4893                 def = data = RREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_CNTL);
4894
4895                 data &= ~(CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY_MASK | CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK);
4896                 data |= (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) |
4897                         (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
4898
4899                 if (def != data)
4900                         WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_CNTL, data);
4901
4902                 data = RREG32_SOC15(GC, 0, regCP_INT_CNTL);
4903                 data = REG_SET_FIELD(data, CP_INT_CNTL, CNTX_BUSY_INT_ENABLE, 1);
4904                 data = REG_SET_FIELD(data, CP_INT_CNTL, CNTX_EMPTY_INT_ENABLE, 1);
4905                 data = REG_SET_FIELD(data, CP_INT_CNTL, CMP_BUSY_INT_ENABLE, 1);
4906                 data = REG_SET_FIELD(data, CP_INT_CNTL, GFX_IDLE_INT_ENABLE, 1);
4907                 WREG32_SOC15(GC, 0, regCP_INT_CNTL, data);
4908
4909                 data = RREG32_SOC15(GC, 0, regSDMA0_RLC_CGCG_CTRL);
4910                 data = REG_SET_FIELD(data, SDMA0_RLC_CGCG_CTRL, CGCG_INT_ENABLE, 1);
4911                 WREG32_SOC15(GC, 0, regSDMA0_RLC_CGCG_CTRL, data);
4912
4913                 /* Some ASICs only have one SDMA instance, not need to configure SDMA1 */
4914                 if (adev->sdma.num_instances > 1) {
4915                         data = RREG32_SOC15(GC, 0, regSDMA1_RLC_CGCG_CTRL);
4916                         data = REG_SET_FIELD(data, SDMA1_RLC_CGCG_CTRL, CGCG_INT_ENABLE, 1);
4917                         WREG32_SOC15(GC, 0, regSDMA1_RLC_CGCG_CTRL, data);
4918                 }
4919         } else {
4920                 /* Program RLC_CGCG_CGLS_CTRL */
4921                 def = data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL);
4922
4923                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)
4924                         data &= ~RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
4925
4926                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
4927                         data &= ~RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;
4928
4929                 if (def != data)
4930                         WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL, data);
4931
4932                 /* Program RLC_CGCG_CGLS_CTRL_3D */
4933                 def = data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D);
4934
4935                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG)
4936                         data &= ~RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK;
4937                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS)
4938                         data &= ~RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK;
4939
4940                 if (def != data)
4941                         WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D, data);
4942
4943                 data = RREG32_SOC15(GC, 0, regSDMA0_RLC_CGCG_CTRL);
4944                 data &= ~SDMA0_RLC_CGCG_CTRL__CGCG_INT_ENABLE_MASK;
4945                 WREG32_SOC15(GC, 0, regSDMA0_RLC_CGCG_CTRL, data);
4946
4947                 /* Some ASICs only have one SDMA instance, not need to configure SDMA1 */
4948                 if (adev->sdma.num_instances > 1) {
4949                         data = RREG32_SOC15(GC, 0, regSDMA1_RLC_CGCG_CTRL);
4950                         data &= ~SDMA1_RLC_CGCG_CTRL__CGCG_INT_ENABLE_MASK;
4951                         WREG32_SOC15(GC, 0, regSDMA1_RLC_CGCG_CTRL, data);
4952                 }
4953         }
4954 }
4955
4956 static int gfx_v11_0_update_gfx_clock_gating(struct amdgpu_device *adev,
4957                                             bool enable)
4958 {
4959         amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
4960
4961         gfx_v11_0_update_coarse_grain_clock_gating(adev, enable);
4962
4963         gfx_v11_0_update_medium_grain_clock_gating(adev, enable);
4964
4965         gfx_v11_0_update_repeater_fgcg(adev, enable);
4966
4967         gfx_v11_0_update_sram_fgcg(adev, enable);
4968
4969         gfx_v11_0_update_perf_clk(adev, enable);
4970
4971         if (adev->cg_flags &
4972             (AMD_CG_SUPPORT_GFX_MGCG |
4973              AMD_CG_SUPPORT_GFX_CGLS |
4974              AMD_CG_SUPPORT_GFX_CGCG |
4975              AMD_CG_SUPPORT_GFX_3D_CGCG |
4976              AMD_CG_SUPPORT_GFX_3D_CGLS))
4977                 gfx_v11_0_enable_gui_idle_interrupt(adev, enable);
4978
4979         amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
4980
4981         return 0;
4982 }
4983
4984 static void gfx_v11_0_update_spm_vmid(struct amdgpu_device *adev, unsigned vmid)
4985 {
4986         u32 data;
4987
4988         amdgpu_gfx_off_ctrl(adev, false);
4989
4990         data = RREG32_SOC15_NO_KIQ(GC, 0, regRLC_SPM_MC_CNTL);
4991
4992         data &= ~RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK;
4993         data |= (vmid & RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK) << RLC_SPM_MC_CNTL__RLC_SPM_VMID__SHIFT;
4994
4995         WREG32_SOC15_NO_KIQ(GC, 0, regRLC_SPM_MC_CNTL, data);
4996
4997         amdgpu_gfx_off_ctrl(adev, true);
4998 }
4999
5000 static const struct amdgpu_rlc_funcs gfx_v11_0_rlc_funcs = {
5001         .is_rlc_enabled = gfx_v11_0_is_rlc_enabled,
5002         .set_safe_mode = gfx_v11_0_set_safe_mode,
5003         .unset_safe_mode = gfx_v11_0_unset_safe_mode,
5004         .init = gfx_v11_0_rlc_init,
5005         .get_csb_size = gfx_v11_0_get_csb_size,
5006         .get_csb_buffer = gfx_v11_0_get_csb_buffer,
5007         .resume = gfx_v11_0_rlc_resume,
5008         .stop = gfx_v11_0_rlc_stop,
5009         .reset = gfx_v11_0_rlc_reset,
5010         .start = gfx_v11_0_rlc_start,
5011         .update_spm_vmid = gfx_v11_0_update_spm_vmid,
5012 };
5013
5014 static void gfx_v11_cntl_power_gating(struct amdgpu_device *adev, bool enable)
5015 {
5016         u32 data = RREG32_SOC15(GC, 0, regRLC_PG_CNTL);
5017
5018         if (enable && (adev->pg_flags & AMD_PG_SUPPORT_GFX_PG))
5019                 data |= RLC_PG_CNTL__GFX_POWER_GATING_ENABLE_MASK;
5020         else
5021                 data &= ~RLC_PG_CNTL__GFX_POWER_GATING_ENABLE_MASK;
5022
5023         WREG32_SOC15(GC, 0, regRLC_PG_CNTL, data);
5024
5025         // Program RLC_PG_DELAY3 for CGPG hysteresis
5026         if (enable && (adev->pg_flags & AMD_PG_SUPPORT_GFX_PG)) {
5027                 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
5028                 case IP_VERSION(11, 0, 1):
5029                 case IP_VERSION(11, 0, 4):
5030                         WREG32_SOC15(GC, 0, regRLC_PG_DELAY_3, RLC_PG_DELAY_3_DEFAULT_GC_11_0_1);
5031                         break;
5032                 default:
5033                         break;
5034                 }
5035         }
5036 }
5037
5038 static void gfx_v11_cntl_pg(struct amdgpu_device *adev, bool enable)
5039 {
5040         amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
5041
5042         gfx_v11_cntl_power_gating(adev, enable);
5043
5044         amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
5045 }
5046
5047 static int gfx_v11_0_set_powergating_state(void *handle,
5048                                            enum amd_powergating_state state)
5049 {
5050         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5051         bool enable = (state == AMD_PG_STATE_GATE);
5052
5053         if (amdgpu_sriov_vf(adev))
5054                 return 0;
5055
5056         switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
5057         case IP_VERSION(11, 0, 0):
5058         case IP_VERSION(11, 0, 2):
5059         case IP_VERSION(11, 0, 3):
5060                 amdgpu_gfx_off_ctrl(adev, enable);
5061                 break;
5062         case IP_VERSION(11, 0, 1):
5063         case IP_VERSION(11, 0, 4):
5064                 if (!enable)
5065                         amdgpu_gfx_off_ctrl(adev, false);
5066
5067                 gfx_v11_cntl_pg(adev, enable);
5068
5069                 if (enable)
5070                         amdgpu_gfx_off_ctrl(adev, true);
5071
5072                 break;
5073         default:
5074                 break;
5075         }
5076
5077         return 0;
5078 }
5079
5080 static int gfx_v11_0_set_clockgating_state(void *handle,
5081                                           enum amd_clockgating_state state)
5082 {
5083         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5084
5085         if (amdgpu_sriov_vf(adev))
5086                 return 0;
5087
5088         switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
5089         case IP_VERSION(11, 0, 0):
5090         case IP_VERSION(11, 0, 1):
5091         case IP_VERSION(11, 0, 2):
5092         case IP_VERSION(11, 0, 3):
5093         case IP_VERSION(11, 0, 4):
5094                 gfx_v11_0_update_gfx_clock_gating(adev,
5095                                 state ==  AMD_CG_STATE_GATE);
5096                 break;
5097         default:
5098                 break;
5099         }
5100
5101         return 0;
5102 }
5103
5104 static void gfx_v11_0_get_clockgating_state(void *handle, u64 *flags)
5105 {
5106         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5107         int data;
5108
5109         /* AMD_CG_SUPPORT_GFX_MGCG */
5110         data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
5111         if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK))
5112                 *flags |= AMD_CG_SUPPORT_GFX_MGCG;
5113
5114         /* AMD_CG_SUPPORT_REPEATER_FGCG */
5115         if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_REPEATER_FGCG_OVERRIDE_MASK))
5116                 *flags |= AMD_CG_SUPPORT_REPEATER_FGCG;
5117
5118         /* AMD_CG_SUPPORT_GFX_FGCG */
5119         if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_FGCG_OVERRIDE_MASK))
5120                 *flags |= AMD_CG_SUPPORT_GFX_FGCG;
5121
5122         /* AMD_CG_SUPPORT_GFX_PERF_CLK */
5123         if (!(data & RLC_CGTT_MGCG_OVERRIDE__PERFMON_CLOCK_STATE_MASK))
5124                 *flags |= AMD_CG_SUPPORT_GFX_PERF_CLK;
5125
5126         /* AMD_CG_SUPPORT_GFX_CGCG */
5127         data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL);
5128         if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK)
5129                 *flags |= AMD_CG_SUPPORT_GFX_CGCG;
5130
5131         /* AMD_CG_SUPPORT_GFX_CGLS */
5132         if (data & RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK)
5133                 *flags |= AMD_CG_SUPPORT_GFX_CGLS;
5134
5135         /* AMD_CG_SUPPORT_GFX_3D_CGCG */
5136         data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D);
5137         if (data & RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK)
5138                 *flags |= AMD_CG_SUPPORT_GFX_3D_CGCG;
5139
5140         /* AMD_CG_SUPPORT_GFX_3D_CGLS */
5141         if (data & RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK)
5142                 *flags |= AMD_CG_SUPPORT_GFX_3D_CGLS;
5143 }
5144
5145 static u64 gfx_v11_0_ring_get_rptr_gfx(struct amdgpu_ring *ring)
5146 {
5147         /* gfx11 is 32bit rptr*/
5148         return *(uint32_t *)ring->rptr_cpu_addr;
5149 }
5150
5151 static u64 gfx_v11_0_ring_get_wptr_gfx(struct amdgpu_ring *ring)
5152 {
5153         struct amdgpu_device *adev = ring->adev;
5154         u64 wptr;
5155
5156         /* XXX check if swapping is necessary on BE */
5157         if (ring->use_doorbell) {
5158                 wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr);
5159         } else {
5160                 wptr = RREG32_SOC15(GC, 0, regCP_RB0_WPTR);
5161                 wptr += (u64)RREG32_SOC15(GC, 0, regCP_RB0_WPTR_HI) << 32;
5162         }
5163
5164         return wptr;
5165 }
5166
5167 static void gfx_v11_0_ring_set_wptr_gfx(struct amdgpu_ring *ring)
5168 {
5169         struct amdgpu_device *adev = ring->adev;
5170         uint32_t *wptr_saved;
5171         uint32_t *is_queue_unmap;
5172         uint64_t aggregated_db_index;
5173         uint32_t mqd_size = adev->mqds[AMDGPU_HW_IP_GFX].mqd_size;
5174         uint64_t wptr_tmp;
5175
5176         if (ring->is_mes_queue) {
5177                 wptr_saved = (uint32_t *)(ring->mqd_ptr + mqd_size);
5178                 is_queue_unmap = (uint32_t *)(ring->mqd_ptr + mqd_size +
5179                                               sizeof(uint32_t));
5180                 aggregated_db_index =
5181                         amdgpu_mes_get_aggregated_doorbell_index(adev,
5182                                                                  ring->hw_prio);
5183
5184                 wptr_tmp = ring->wptr & ring->buf_mask;
5185                 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, wptr_tmp);
5186                 *wptr_saved = wptr_tmp;
5187                 /* assume doorbell always being used by mes mapped queue */
5188                 if (*is_queue_unmap) {
5189                         WDOORBELL64(aggregated_db_index, wptr_tmp);
5190                         WDOORBELL64(ring->doorbell_index, wptr_tmp);
5191                 } else {
5192                         WDOORBELL64(ring->doorbell_index, wptr_tmp);
5193
5194                         if (*is_queue_unmap)
5195                                 WDOORBELL64(aggregated_db_index, wptr_tmp);
5196                 }
5197         } else {
5198                 if (ring->use_doorbell) {
5199                         /* XXX check if swapping is necessary on BE */
5200                         atomic64_set((atomic64_t *)ring->wptr_cpu_addr,
5201                                      ring->wptr);
5202                         WDOORBELL64(ring->doorbell_index, ring->wptr);
5203                 } else {
5204                         WREG32_SOC15(GC, 0, regCP_RB0_WPTR,
5205                                      lower_32_bits(ring->wptr));
5206                         WREG32_SOC15(GC, 0, regCP_RB0_WPTR_HI,
5207                                      upper_32_bits(ring->wptr));
5208                 }
5209         }
5210 }
5211
5212 static u64 gfx_v11_0_ring_get_rptr_compute(struct amdgpu_ring *ring)
5213 {
5214         /* gfx11 hardware is 32bit rptr */
5215         return *(uint32_t *)ring->rptr_cpu_addr;
5216 }
5217
5218 static u64 gfx_v11_0_ring_get_wptr_compute(struct amdgpu_ring *ring)
5219 {
5220         u64 wptr;
5221
5222         /* XXX check if swapping is necessary on BE */
5223         if (ring->use_doorbell)
5224                 wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr);
5225         else
5226                 BUG();
5227         return wptr;
5228 }
5229
5230 static void gfx_v11_0_ring_set_wptr_compute(struct amdgpu_ring *ring)
5231 {
5232         struct amdgpu_device *adev = ring->adev;
5233         uint32_t *wptr_saved;
5234         uint32_t *is_queue_unmap;
5235         uint64_t aggregated_db_index;
5236         uint32_t mqd_size = adev->mqds[AMDGPU_HW_IP_COMPUTE].mqd_size;
5237         uint64_t wptr_tmp;
5238
5239         if (ring->is_mes_queue) {
5240                 wptr_saved = (uint32_t *)(ring->mqd_ptr + mqd_size);
5241                 is_queue_unmap = (uint32_t *)(ring->mqd_ptr + mqd_size +
5242                                               sizeof(uint32_t));
5243                 aggregated_db_index =
5244                         amdgpu_mes_get_aggregated_doorbell_index(adev,
5245                                                                  ring->hw_prio);
5246
5247                 wptr_tmp = ring->wptr & ring->buf_mask;
5248                 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, wptr_tmp);
5249                 *wptr_saved = wptr_tmp;
5250                 /* assume doorbell always used by mes mapped queue */
5251                 if (*is_queue_unmap) {
5252                         WDOORBELL64(aggregated_db_index, wptr_tmp);
5253                         WDOORBELL64(ring->doorbell_index, wptr_tmp);
5254                 } else {
5255                         WDOORBELL64(ring->doorbell_index, wptr_tmp);
5256
5257                         if (*is_queue_unmap)
5258                                 WDOORBELL64(aggregated_db_index, wptr_tmp);
5259                 }
5260         } else {
5261                 /* XXX check if swapping is necessary on BE */
5262                 if (ring->use_doorbell) {
5263                         atomic64_set((atomic64_t *)ring->wptr_cpu_addr,
5264                                      ring->wptr);
5265                         WDOORBELL64(ring->doorbell_index, ring->wptr);
5266                 } else {
5267                         BUG(); /* only DOORBELL method supported on gfx11 now */
5268                 }
5269         }
5270 }
5271
5272 static void gfx_v11_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
5273 {
5274         struct amdgpu_device *adev = ring->adev;
5275         u32 ref_and_mask, reg_mem_engine;
5276         const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg;
5277
5278         if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
5279                 switch (ring->me) {
5280                 case 1:
5281                         ref_and_mask = nbio_hf_reg->ref_and_mask_cp2 << ring->pipe;
5282                         break;
5283                 case 2:
5284                         ref_and_mask = nbio_hf_reg->ref_and_mask_cp6 << ring->pipe;
5285                         break;
5286                 default:
5287                         return;
5288                 }
5289                 reg_mem_engine = 0;
5290         } else {
5291                 ref_and_mask = nbio_hf_reg->ref_and_mask_cp0;
5292                 reg_mem_engine = 1; /* pfp */
5293         }
5294
5295         gfx_v11_0_wait_reg_mem(ring, reg_mem_engine, 0, 1,
5296                                adev->nbio.funcs->get_hdp_flush_req_offset(adev),
5297                                adev->nbio.funcs->get_hdp_flush_done_offset(adev),
5298                                ref_and_mask, ref_and_mask, 0x20);
5299 }
5300
5301 static void gfx_v11_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
5302                                        struct amdgpu_job *job,
5303                                        struct amdgpu_ib *ib,
5304                                        uint32_t flags)
5305 {
5306         unsigned vmid = AMDGPU_JOB_GET_VMID(job);
5307         u32 header, control = 0;
5308
5309         BUG_ON(ib->flags & AMDGPU_IB_FLAG_CE);
5310
5311         header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
5312
5313         control |= ib->length_dw | (vmid << 24);
5314
5315         if (ring->adev->gfx.mcbp && (ib->flags & AMDGPU_IB_FLAG_PREEMPT)) {
5316                 control |= INDIRECT_BUFFER_PRE_ENB(1);
5317
5318                 if (flags & AMDGPU_IB_PREEMPTED)
5319                         control |= INDIRECT_BUFFER_PRE_RESUME(1);
5320
5321                 if (vmid)
5322                         gfx_v11_0_ring_emit_de_meta(ring,
5323                                     (!amdgpu_sriov_vf(ring->adev) && flags & AMDGPU_IB_PREEMPTED) ? true : false);
5324         }
5325
5326         if (ring->is_mes_queue)
5327                 /* inherit vmid from mqd */
5328                 control |= 0x400000;
5329
5330         amdgpu_ring_write(ring, header);
5331         BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
5332         amdgpu_ring_write(ring,
5333 #ifdef __BIG_ENDIAN
5334                 (2 << 0) |
5335 #endif
5336                 lower_32_bits(ib->gpu_addr));
5337         amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
5338         amdgpu_ring_write(ring, control);
5339 }
5340
5341 static void gfx_v11_0_ring_emit_ib_compute(struct amdgpu_ring *ring,
5342                                            struct amdgpu_job *job,
5343                                            struct amdgpu_ib *ib,
5344                                            uint32_t flags)
5345 {
5346         unsigned vmid = AMDGPU_JOB_GET_VMID(job);
5347         u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24);
5348
5349         if (ring->is_mes_queue)
5350                 /* inherit vmid from mqd */
5351                 control |= 0x40000000;
5352
5353         /* Currently, there is a high possibility to get wave ID mismatch
5354          * between ME and GDS, leading to a hw deadlock, because ME generates
5355          * different wave IDs than the GDS expects. This situation happens
5356          * randomly when at least 5 compute pipes use GDS ordered append.
5357          * The wave IDs generated by ME are also wrong after suspend/resume.
5358          * Those are probably bugs somewhere else in the kernel driver.
5359          *
5360          * Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and
5361          * GDS to 0 for this ring (me/pipe).
5362          */
5363         if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) {
5364                 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
5365                 amdgpu_ring_write(ring, regGDS_COMPUTE_MAX_WAVE_ID);
5366                 amdgpu_ring_write(ring, ring->adev->gds.gds_compute_max_wave_id);
5367         }
5368
5369         amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
5370         BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
5371         amdgpu_ring_write(ring,
5372 #ifdef __BIG_ENDIAN
5373                                 (2 << 0) |
5374 #endif
5375                                 lower_32_bits(ib->gpu_addr));
5376         amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
5377         amdgpu_ring_write(ring, control);
5378 }
5379
5380 static void gfx_v11_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
5381                                      u64 seq, unsigned flags)
5382 {
5383         bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
5384         bool int_sel = flags & AMDGPU_FENCE_FLAG_INT;
5385
5386         /* RELEASE_MEM - flush caches, send int */
5387         amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 6));
5388         amdgpu_ring_write(ring, (PACKET3_RELEASE_MEM_GCR_SEQ |
5389                                  PACKET3_RELEASE_MEM_GCR_GL2_WB |
5390                                  PACKET3_RELEASE_MEM_GCR_GL2_INV |
5391                                  PACKET3_RELEASE_MEM_GCR_GL2_US |
5392                                  PACKET3_RELEASE_MEM_GCR_GL1_INV |
5393                                  PACKET3_RELEASE_MEM_GCR_GLV_INV |
5394                                  PACKET3_RELEASE_MEM_GCR_GLM_INV |
5395                                  PACKET3_RELEASE_MEM_GCR_GLM_WB |
5396                                  PACKET3_RELEASE_MEM_CACHE_POLICY(3) |
5397                                  PACKET3_RELEASE_MEM_EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
5398                                  PACKET3_RELEASE_MEM_EVENT_INDEX(5)));
5399         amdgpu_ring_write(ring, (PACKET3_RELEASE_MEM_DATA_SEL(write64bit ? 2 : 1) |
5400                                  PACKET3_RELEASE_MEM_INT_SEL(int_sel ? 2 : 0)));
5401
5402         /*
5403          * the address should be Qword aligned if 64bit write, Dword
5404          * aligned if only send 32bit data low (discard data high)
5405          */
5406         if (write64bit)
5407                 BUG_ON(addr & 0x7);
5408         else
5409                 BUG_ON(addr & 0x3);
5410         amdgpu_ring_write(ring, lower_32_bits(addr));
5411         amdgpu_ring_write(ring, upper_32_bits(addr));
5412         amdgpu_ring_write(ring, lower_32_bits(seq));
5413         amdgpu_ring_write(ring, upper_32_bits(seq));
5414         amdgpu_ring_write(ring, ring->is_mes_queue ?
5415                          (ring->hw_queue_id | AMDGPU_FENCE_MES_QUEUE_FLAG) : 0);
5416 }
5417
5418 static void gfx_v11_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
5419 {
5420         int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
5421         uint32_t seq = ring->fence_drv.sync_seq;
5422         uint64_t addr = ring->fence_drv.gpu_addr;
5423
5424         gfx_v11_0_wait_reg_mem(ring, usepfp, 1, 0, lower_32_bits(addr),
5425                                upper_32_bits(addr), seq, 0xffffffff, 4);
5426 }
5427
5428 static void gfx_v11_0_ring_invalidate_tlbs(struct amdgpu_ring *ring,
5429                                    uint16_t pasid, uint32_t flush_type,
5430                                    bool all_hub, uint8_t dst_sel)
5431 {
5432         amdgpu_ring_write(ring, PACKET3(PACKET3_INVALIDATE_TLBS, 0));
5433         amdgpu_ring_write(ring,
5434                           PACKET3_INVALIDATE_TLBS_DST_SEL(dst_sel) |
5435                           PACKET3_INVALIDATE_TLBS_ALL_HUB(all_hub) |
5436                           PACKET3_INVALIDATE_TLBS_PASID(pasid) |
5437                           PACKET3_INVALIDATE_TLBS_FLUSH_TYPE(flush_type));
5438 }
5439
5440 static void gfx_v11_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
5441                                          unsigned vmid, uint64_t pd_addr)
5442 {
5443         if (ring->is_mes_queue)
5444                 gfx_v11_0_ring_invalidate_tlbs(ring, 0, 0, false, 0);
5445         else
5446                 amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
5447
5448         /* compute doesn't have PFP */
5449         if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) {
5450                 /* sync PFP to ME, otherwise we might get invalid PFP reads */
5451                 amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5452                 amdgpu_ring_write(ring, 0x0);
5453         }
5454 }
5455
5456 static void gfx_v11_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr,
5457                                           u64 seq, unsigned int flags)
5458 {
5459         struct amdgpu_device *adev = ring->adev;
5460
5461         /* we only allocate 32bit for each seq wb address */
5462         BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
5463
5464         /* write fence seq to the "addr" */
5465         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5466         amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5467                                  WRITE_DATA_DST_SEL(5) | WR_CONFIRM));
5468         amdgpu_ring_write(ring, lower_32_bits(addr));
5469         amdgpu_ring_write(ring, upper_32_bits(addr));
5470         amdgpu_ring_write(ring, lower_32_bits(seq));
5471
5472         if (flags & AMDGPU_FENCE_FLAG_INT) {
5473                 /* set register to trigger INT */
5474                 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5475                 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5476                                          WRITE_DATA_DST_SEL(0) | WR_CONFIRM));
5477                 amdgpu_ring_write(ring, SOC15_REG_OFFSET(GC, 0, regCPC_INT_STATUS));
5478                 amdgpu_ring_write(ring, 0);
5479                 amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */
5480         }
5481 }
5482
5483 static void gfx_v11_0_ring_emit_cntxcntl(struct amdgpu_ring *ring,
5484                                          uint32_t flags)
5485 {
5486         uint32_t dw2 = 0;
5487
5488         dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */
5489         if (flags & AMDGPU_HAVE_CTX_SWITCH) {
5490                 /* set load_global_config & load_global_uconfig */
5491                 dw2 |= 0x8001;
5492                 /* set load_cs_sh_regs */
5493                 dw2 |= 0x01000000;
5494                 /* set load_per_context_state & load_gfx_sh_regs for GFX */
5495                 dw2 |= 0x10002;
5496         }
5497
5498         amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
5499         amdgpu_ring_write(ring, dw2);
5500         amdgpu_ring_write(ring, 0);
5501 }
5502
5503 static void gfx_v11_0_ring_emit_gfx_shadow(struct amdgpu_ring *ring,
5504                                            u64 shadow_va, u64 csa_va,
5505                                            u64 gds_va, bool init_shadow,
5506                                            int vmid)
5507 {
5508         struct amdgpu_device *adev = ring->adev;
5509
5510         if (!adev->gfx.cp_gfx_shadow)
5511                 return;
5512
5513         amdgpu_ring_write(ring, PACKET3(PACKET3_SET_Q_PREEMPTION_MODE, 7));
5514         amdgpu_ring_write(ring, lower_32_bits(shadow_va));
5515         amdgpu_ring_write(ring, upper_32_bits(shadow_va));
5516         amdgpu_ring_write(ring, lower_32_bits(gds_va));
5517         amdgpu_ring_write(ring, upper_32_bits(gds_va));
5518         amdgpu_ring_write(ring, lower_32_bits(csa_va));
5519         amdgpu_ring_write(ring, upper_32_bits(csa_va));
5520         amdgpu_ring_write(ring, shadow_va ?
5521                           PACKET3_SET_Q_PREEMPTION_MODE_IB_VMID(vmid) : 0);
5522         amdgpu_ring_write(ring, init_shadow ?
5523                           PACKET3_SET_Q_PREEMPTION_MODE_INIT_SHADOW_MEM : 0);
5524 }
5525
5526 static unsigned gfx_v11_0_ring_emit_init_cond_exec(struct amdgpu_ring *ring)
5527 {
5528         unsigned ret;
5529
5530         amdgpu_ring_write(ring, PACKET3(PACKET3_COND_EXEC, 3));
5531         amdgpu_ring_write(ring, lower_32_bits(ring->cond_exe_gpu_addr));
5532         amdgpu_ring_write(ring, upper_32_bits(ring->cond_exe_gpu_addr));
5533         amdgpu_ring_write(ring, 0); /* discard following DWs if *cond_exec_gpu_addr==0 */
5534         ret = ring->wptr & ring->buf_mask;
5535         amdgpu_ring_write(ring, 0x55aa55aa); /* patch dummy value later */
5536
5537         return ret;
5538 }
5539
5540 static void gfx_v11_0_ring_emit_patch_cond_exec(struct amdgpu_ring *ring, unsigned offset)
5541 {
5542         unsigned cur;
5543         BUG_ON(offset > ring->buf_mask);
5544         BUG_ON(ring->ring[offset] != 0x55aa55aa);
5545
5546         cur = (ring->wptr - 1) & ring->buf_mask;
5547         if (likely(cur > offset))
5548                 ring->ring[offset] = cur - offset;
5549         else
5550                 ring->ring[offset] = (ring->buf_mask + 1) - offset + cur;
5551 }
5552
5553 static int gfx_v11_0_ring_preempt_ib(struct amdgpu_ring *ring)
5554 {
5555         int i, r = 0;
5556         struct amdgpu_device *adev = ring->adev;
5557         struct amdgpu_kiq *kiq = &adev->gfx.kiq[0];
5558         struct amdgpu_ring *kiq_ring = &kiq->ring;
5559         unsigned long flags;
5560
5561         if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues)
5562                 return -EINVAL;
5563
5564         spin_lock_irqsave(&kiq->ring_lock, flags);
5565
5566         if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size)) {
5567                 spin_unlock_irqrestore(&kiq->ring_lock, flags);
5568                 return -ENOMEM;
5569         }
5570
5571         /* assert preemption condition */
5572         amdgpu_ring_set_preempt_cond_exec(ring, false);
5573
5574         /* assert IB preemption, emit the trailing fence */
5575         kiq->pmf->kiq_unmap_queues(kiq_ring, ring, PREEMPT_QUEUES_NO_UNMAP,
5576                                    ring->trail_fence_gpu_addr,
5577                                    ++ring->trail_seq);
5578         amdgpu_ring_commit(kiq_ring);
5579
5580         spin_unlock_irqrestore(&kiq->ring_lock, flags);
5581
5582         /* poll the trailing fence */
5583         for (i = 0; i < adev->usec_timeout; i++) {
5584                 if (ring->trail_seq ==
5585                     le32_to_cpu(*(ring->trail_fence_cpu_addr)))
5586                         break;
5587                 udelay(1);
5588         }
5589
5590         if (i >= adev->usec_timeout) {
5591                 r = -EINVAL;
5592                 DRM_ERROR("ring %d failed to preempt ib\n", ring->idx);
5593         }
5594
5595         /* deassert preemption condition */
5596         amdgpu_ring_set_preempt_cond_exec(ring, true);
5597         return r;
5598 }
5599
5600 static void gfx_v11_0_ring_emit_de_meta(struct amdgpu_ring *ring, bool resume)
5601 {
5602         struct amdgpu_device *adev = ring->adev;
5603         struct v10_de_ib_state de_payload = {0};
5604         uint64_t offset, gds_addr, de_payload_gpu_addr;
5605         void *de_payload_cpu_addr;
5606         int cnt;
5607
5608         if (ring->is_mes_queue) {
5609                 offset = offsetof(struct amdgpu_mes_ctx_meta_data,
5610                                   gfx[0].gfx_meta_data) +
5611                         offsetof(struct v10_gfx_meta_data, de_payload);
5612                 de_payload_gpu_addr =
5613                         amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset);
5614                 de_payload_cpu_addr =
5615                         amdgpu_mes_ctx_get_offs_cpu_addr(ring, offset);
5616
5617                 offset = offsetof(struct amdgpu_mes_ctx_meta_data,
5618                                   gfx[0].gds_backup) +
5619                         offsetof(struct v10_gfx_meta_data, de_payload);
5620                 gds_addr = amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset);
5621         } else {
5622                 offset = offsetof(struct v10_gfx_meta_data, de_payload);
5623                 de_payload_gpu_addr = amdgpu_csa_vaddr(ring->adev) + offset;
5624                 de_payload_cpu_addr = adev->virt.csa_cpu_addr + offset;
5625
5626                 gds_addr = ALIGN(amdgpu_csa_vaddr(ring->adev) +
5627                                  AMDGPU_CSA_SIZE - adev->gds.gds_size,
5628                                  PAGE_SIZE);
5629         }
5630
5631         de_payload.gds_backup_addrlo = lower_32_bits(gds_addr);
5632         de_payload.gds_backup_addrhi = upper_32_bits(gds_addr);
5633
5634         cnt = (sizeof(de_payload) >> 2) + 4 - 2;
5635         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt));
5636         amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
5637                                  WRITE_DATA_DST_SEL(8) |
5638                                  WR_CONFIRM) |
5639                                  WRITE_DATA_CACHE_POLICY(0));
5640         amdgpu_ring_write(ring, lower_32_bits(de_payload_gpu_addr));
5641         amdgpu_ring_write(ring, upper_32_bits(de_payload_gpu_addr));
5642
5643         if (resume)
5644                 amdgpu_ring_write_multiple(ring, de_payload_cpu_addr,
5645                                            sizeof(de_payload) >> 2);
5646         else
5647                 amdgpu_ring_write_multiple(ring, (void *)&de_payload,
5648                                            sizeof(de_payload) >> 2);
5649 }
5650
5651 static void gfx_v11_0_ring_emit_frame_cntl(struct amdgpu_ring *ring, bool start,
5652                                     bool secure)
5653 {
5654         uint32_t v = secure ? FRAME_TMZ : 0;
5655
5656         amdgpu_ring_write(ring, PACKET3(PACKET3_FRAME_CONTROL, 0));
5657         amdgpu_ring_write(ring, v | FRAME_CMD(start ? 0 : 1));
5658 }
5659
5660 static void gfx_v11_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg,
5661                                      uint32_t reg_val_offs)
5662 {
5663         struct amdgpu_device *adev = ring->adev;
5664
5665         amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));
5666         amdgpu_ring_write(ring, 0 |     /* src: register*/
5667                                 (5 << 8) |      /* dst: memory */
5668                                 (1 << 20));     /* write confirm */
5669         amdgpu_ring_write(ring, reg);
5670         amdgpu_ring_write(ring, 0);
5671         amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +
5672                                 reg_val_offs * 4));
5673         amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +
5674                                 reg_val_offs * 4));
5675 }
5676
5677 static void gfx_v11_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg,
5678                                    uint32_t val)
5679 {
5680         uint32_t cmd = 0;
5681
5682         switch (ring->funcs->type) {
5683         case AMDGPU_RING_TYPE_GFX:
5684                 cmd = WRITE_DATA_ENGINE_SEL(1) | WR_CONFIRM;
5685                 break;
5686         case AMDGPU_RING_TYPE_KIQ:
5687                 cmd = (1 << 16); /* no inc addr */
5688                 break;
5689         default:
5690                 cmd = WR_CONFIRM;
5691                 break;
5692         }
5693         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5694         amdgpu_ring_write(ring, cmd);
5695         amdgpu_ring_write(ring, reg);
5696         amdgpu_ring_write(ring, 0);
5697         amdgpu_ring_write(ring, val);
5698 }
5699
5700 static void gfx_v11_0_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
5701                                         uint32_t val, uint32_t mask)
5702 {
5703         gfx_v11_0_wait_reg_mem(ring, 0, 0, 0, reg, 0, val, mask, 0x20);
5704 }
5705
5706 static void gfx_v11_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring,
5707                                                    uint32_t reg0, uint32_t reg1,
5708                                                    uint32_t ref, uint32_t mask)
5709 {
5710         int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
5711
5712         gfx_v11_0_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1,
5713                                ref, mask, 0x20);
5714 }
5715
5716 static void gfx_v11_0_ring_soft_recovery(struct amdgpu_ring *ring,
5717                                          unsigned vmid)
5718 {
5719         struct amdgpu_device *adev = ring->adev;
5720         uint32_t value = 0;
5721
5722         value = REG_SET_FIELD(value, SQ_CMD, CMD, 0x03);
5723         value = REG_SET_FIELD(value, SQ_CMD, MODE, 0x01);
5724         value = REG_SET_FIELD(value, SQ_CMD, CHECK_VMID, 1);
5725         value = REG_SET_FIELD(value, SQ_CMD, VM_ID, vmid);
5726         WREG32_SOC15(GC, 0, regSQ_CMD, value);
5727 }
5728
5729 static void
5730 gfx_v11_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev,
5731                                       uint32_t me, uint32_t pipe,
5732                                       enum amdgpu_interrupt_state state)
5733 {
5734         uint32_t cp_int_cntl, cp_int_cntl_reg;
5735
5736         if (!me) {
5737                 switch (pipe) {
5738                 case 0:
5739                         cp_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_INT_CNTL_RING0);
5740                         break;
5741                 case 1:
5742                         cp_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_INT_CNTL_RING1);
5743                         break;
5744                 default:
5745                         DRM_DEBUG("invalid pipe %d\n", pipe);
5746                         return;
5747                 }
5748         } else {
5749                 DRM_DEBUG("invalid me %d\n", me);
5750                 return;
5751         }
5752
5753         switch (state) {
5754         case AMDGPU_IRQ_STATE_DISABLE:
5755                 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg);
5756                 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
5757                                             TIME_STAMP_INT_ENABLE, 0);
5758                 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
5759                                             GENERIC0_INT_ENABLE, 0);
5760                 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl);
5761                 break;
5762         case AMDGPU_IRQ_STATE_ENABLE:
5763                 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg);
5764                 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
5765                                             TIME_STAMP_INT_ENABLE, 1);
5766                 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
5767                                             GENERIC0_INT_ENABLE, 1);
5768                 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl);
5769                 break;
5770         default:
5771                 break;
5772         }
5773 }
5774
5775 static void gfx_v11_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev,
5776                                                      int me, int pipe,
5777                                                      enum amdgpu_interrupt_state state)
5778 {
5779         u32 mec_int_cntl, mec_int_cntl_reg;
5780
5781         /*
5782          * amdgpu controls only the first MEC. That's why this function only
5783          * handles the setting of interrupts for this specific MEC. All other
5784          * pipes' interrupts are set by amdkfd.
5785          */
5786
5787         if (me == 1) {
5788                 switch (pipe) {
5789                 case 0:
5790                         mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE0_INT_CNTL);
5791                         break;
5792                 case 1:
5793                         mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE1_INT_CNTL);
5794                         break;
5795                 case 2:
5796                         mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE2_INT_CNTL);
5797                         break;
5798                 case 3:
5799                         mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE3_INT_CNTL);
5800                         break;
5801                 default:
5802                         DRM_DEBUG("invalid pipe %d\n", pipe);
5803                         return;
5804                 }
5805         } else {
5806                 DRM_DEBUG("invalid me %d\n", me);
5807                 return;
5808         }
5809
5810         switch (state) {
5811         case AMDGPU_IRQ_STATE_DISABLE:
5812                 mec_int_cntl = RREG32_SOC15_IP(GC, mec_int_cntl_reg);
5813                 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
5814                                              TIME_STAMP_INT_ENABLE, 0);
5815                 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
5816                                              GENERIC0_INT_ENABLE, 0);
5817                 WREG32_SOC15_IP(GC, mec_int_cntl_reg, mec_int_cntl);
5818                 break;
5819         case AMDGPU_IRQ_STATE_ENABLE:
5820                 mec_int_cntl = RREG32_SOC15_IP(GC, mec_int_cntl_reg);
5821                 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
5822                                              TIME_STAMP_INT_ENABLE, 1);
5823                 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
5824                                              GENERIC0_INT_ENABLE, 1);
5825                 WREG32_SOC15_IP(GC, mec_int_cntl_reg, mec_int_cntl);
5826                 break;
5827         default:
5828                 break;
5829         }
5830 }
5831
5832 static int gfx_v11_0_set_eop_interrupt_state(struct amdgpu_device *adev,
5833                                             struct amdgpu_irq_src *src,
5834                                             unsigned type,
5835                                             enum amdgpu_interrupt_state state)
5836 {
5837         switch (type) {
5838         case AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP:
5839                 gfx_v11_0_set_gfx_eop_interrupt_state(adev, 0, 0, state);
5840                 break;
5841         case AMDGPU_CP_IRQ_GFX_ME0_PIPE1_EOP:
5842                 gfx_v11_0_set_gfx_eop_interrupt_state(adev, 0, 1, state);
5843                 break;
5844         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP:
5845                 gfx_v11_0_set_compute_eop_interrupt_state(adev, 1, 0, state);
5846                 break;
5847         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP:
5848                 gfx_v11_0_set_compute_eop_interrupt_state(adev, 1, 1, state);
5849                 break;
5850         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP:
5851                 gfx_v11_0_set_compute_eop_interrupt_state(adev, 1, 2, state);
5852                 break;
5853         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP:
5854                 gfx_v11_0_set_compute_eop_interrupt_state(adev, 1, 3, state);
5855                 break;
5856         default:
5857                 break;
5858         }
5859         return 0;
5860 }
5861
5862 static int gfx_v11_0_eop_irq(struct amdgpu_device *adev,
5863                              struct amdgpu_irq_src *source,
5864                              struct amdgpu_iv_entry *entry)
5865 {
5866         int i;
5867         u8 me_id, pipe_id, queue_id;
5868         struct amdgpu_ring *ring;
5869         uint32_t mes_queue_id = entry->src_data[0];
5870
5871         DRM_DEBUG("IH: CP EOP\n");
5872
5873         if (adev->enable_mes && (mes_queue_id & AMDGPU_FENCE_MES_QUEUE_FLAG)) {
5874                 struct amdgpu_mes_queue *queue;
5875
5876                 mes_queue_id &= AMDGPU_FENCE_MES_QUEUE_ID_MASK;
5877
5878                 spin_lock(&adev->mes.queue_id_lock);
5879                 queue = idr_find(&adev->mes.queue_id_idr, mes_queue_id);
5880                 if (queue) {
5881                         DRM_DEBUG("process mes queue id = %d\n", mes_queue_id);
5882                         amdgpu_fence_process(queue->ring);
5883                 }
5884                 spin_unlock(&adev->mes.queue_id_lock);
5885         } else {
5886                 me_id = (entry->ring_id & 0x0c) >> 2;
5887                 pipe_id = (entry->ring_id & 0x03) >> 0;
5888                 queue_id = (entry->ring_id & 0x70) >> 4;
5889
5890                 switch (me_id) {
5891                 case 0:
5892                         if (pipe_id == 0)
5893                                 amdgpu_fence_process(&adev->gfx.gfx_ring[0]);
5894                         else
5895                                 amdgpu_fence_process(&adev->gfx.gfx_ring[1]);
5896                         break;
5897                 case 1:
5898                 case 2:
5899                         for (i = 0; i < adev->gfx.num_compute_rings; i++) {
5900                                 ring = &adev->gfx.compute_ring[i];
5901                                 /* Per-queue interrupt is supported for MEC starting from VI.
5902                                  * The interrupt can only be enabled/disabled per pipe instead
5903                                  * of per queue.
5904                                  */
5905                                 if ((ring->me == me_id) &&
5906                                     (ring->pipe == pipe_id) &&
5907                                     (ring->queue == queue_id))
5908                                         amdgpu_fence_process(ring);
5909                         }
5910                         break;
5911                 }
5912         }
5913
5914         return 0;
5915 }
5916
5917 static int gfx_v11_0_set_priv_reg_fault_state(struct amdgpu_device *adev,
5918                                               struct amdgpu_irq_src *source,
5919                                               unsigned type,
5920                                               enum amdgpu_interrupt_state state)
5921 {
5922         switch (state) {
5923         case AMDGPU_IRQ_STATE_DISABLE:
5924         case AMDGPU_IRQ_STATE_ENABLE:
5925                 WREG32_FIELD15_PREREG(GC, 0, CP_INT_CNTL_RING0,
5926                                PRIV_REG_INT_ENABLE,
5927                                state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
5928                 break;
5929         default:
5930                 break;
5931         }
5932
5933         return 0;
5934 }
5935
5936 static int gfx_v11_0_set_priv_inst_fault_state(struct amdgpu_device *adev,
5937                                                struct amdgpu_irq_src *source,
5938                                                unsigned type,
5939                                                enum amdgpu_interrupt_state state)
5940 {
5941         switch (state) {
5942         case AMDGPU_IRQ_STATE_DISABLE:
5943         case AMDGPU_IRQ_STATE_ENABLE:
5944                 WREG32_FIELD15_PREREG(GC, 0, CP_INT_CNTL_RING0,
5945                                PRIV_INSTR_INT_ENABLE,
5946                                state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
5947                 break;
5948         default:
5949                 break;
5950         }
5951
5952         return 0;
5953 }
5954
5955 static void gfx_v11_0_handle_priv_fault(struct amdgpu_device *adev,
5956                                         struct amdgpu_iv_entry *entry)
5957 {
5958         u8 me_id, pipe_id, queue_id;
5959         struct amdgpu_ring *ring;
5960         int i;
5961
5962         me_id = (entry->ring_id & 0x0c) >> 2;
5963         pipe_id = (entry->ring_id & 0x03) >> 0;
5964         queue_id = (entry->ring_id & 0x70) >> 4;
5965
5966         switch (me_id) {
5967         case 0:
5968                 for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
5969                         ring = &adev->gfx.gfx_ring[i];
5970                         /* we only enabled 1 gfx queue per pipe for now */
5971                         if (ring->me == me_id && ring->pipe == pipe_id)
5972                                 drm_sched_fault(&ring->sched);
5973                 }
5974                 break;
5975         case 1:
5976         case 2:
5977                 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
5978                         ring = &adev->gfx.compute_ring[i];
5979                         if (ring->me == me_id && ring->pipe == pipe_id &&
5980                             ring->queue == queue_id)
5981                                 drm_sched_fault(&ring->sched);
5982                 }
5983                 break;
5984         default:
5985                 BUG();
5986                 break;
5987         }
5988 }
5989
5990 static int gfx_v11_0_priv_reg_irq(struct amdgpu_device *adev,
5991                                   struct amdgpu_irq_src *source,
5992                                   struct amdgpu_iv_entry *entry)
5993 {
5994         DRM_ERROR("Illegal register access in command stream\n");
5995         gfx_v11_0_handle_priv_fault(adev, entry);
5996         return 0;
5997 }
5998
5999 static int gfx_v11_0_priv_inst_irq(struct amdgpu_device *adev,
6000                                    struct amdgpu_irq_src *source,
6001                                    struct amdgpu_iv_entry *entry)
6002 {
6003         DRM_ERROR("Illegal instruction in command stream\n");
6004         gfx_v11_0_handle_priv_fault(adev, entry);
6005         return 0;
6006 }
6007
6008 static int gfx_v11_0_rlc_gc_fed_irq(struct amdgpu_device *adev,
6009                                   struct amdgpu_irq_src *source,
6010                                   struct amdgpu_iv_entry *entry)
6011 {
6012         if (adev->gfx.ras && adev->gfx.ras->rlc_gc_fed_irq)
6013                 return adev->gfx.ras->rlc_gc_fed_irq(adev, source, entry);
6014
6015         return 0;
6016 }
6017
6018 #if 0
6019 static int gfx_v11_0_kiq_set_interrupt_state(struct amdgpu_device *adev,
6020                                              struct amdgpu_irq_src *src,
6021                                              unsigned int type,
6022                                              enum amdgpu_interrupt_state state)
6023 {
6024         uint32_t tmp, target;
6025         struct amdgpu_ring *ring = &(adev->gfx.kiq[0].ring);
6026
6027         target = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE0_INT_CNTL);
6028         target += ring->pipe;
6029
6030         switch (type) {
6031         case AMDGPU_CP_KIQ_IRQ_DRIVER0:
6032                 if (state == AMDGPU_IRQ_STATE_DISABLE) {
6033                         tmp = RREG32_SOC15(GC, 0, regCPC_INT_CNTL);
6034                         tmp = REG_SET_FIELD(tmp, CPC_INT_CNTL,
6035                                             GENERIC2_INT_ENABLE, 0);
6036                         WREG32_SOC15(GC, 0, regCPC_INT_CNTL, tmp);
6037
6038                         tmp = RREG32_SOC15_IP(GC, target);
6039                         tmp = REG_SET_FIELD(tmp, CP_ME1_PIPE0_INT_CNTL,
6040                                             GENERIC2_INT_ENABLE, 0);
6041                         WREG32_SOC15_IP(GC, target, tmp);
6042                 } else {
6043                         tmp = RREG32_SOC15(GC, 0, regCPC_INT_CNTL);
6044                         tmp = REG_SET_FIELD(tmp, CPC_INT_CNTL,
6045                                             GENERIC2_INT_ENABLE, 1);
6046                         WREG32_SOC15(GC, 0, regCPC_INT_CNTL, tmp);
6047
6048                         tmp = RREG32_SOC15_IP(GC, target);
6049                         tmp = REG_SET_FIELD(tmp, CP_ME1_PIPE0_INT_CNTL,
6050                                             GENERIC2_INT_ENABLE, 1);
6051                         WREG32_SOC15_IP(GC, target, tmp);
6052                 }
6053                 break;
6054         default:
6055                 BUG(); /* kiq only support GENERIC2_INT now */
6056                 break;
6057         }
6058         return 0;
6059 }
6060 #endif
6061
6062 static void gfx_v11_0_emit_mem_sync(struct amdgpu_ring *ring)
6063 {
6064         const unsigned int gcr_cntl =
6065                         PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_INV(1) |
6066                         PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_WB(1) |
6067                         PACKET3_ACQUIRE_MEM_GCR_CNTL_GLM_INV(1) |
6068                         PACKET3_ACQUIRE_MEM_GCR_CNTL_GLM_WB(1) |
6069                         PACKET3_ACQUIRE_MEM_GCR_CNTL_GL1_INV(1) |
6070                         PACKET3_ACQUIRE_MEM_GCR_CNTL_GLV_INV(1) |
6071                         PACKET3_ACQUIRE_MEM_GCR_CNTL_GLK_INV(1) |
6072                         PACKET3_ACQUIRE_MEM_GCR_CNTL_GLI_INV(1);
6073
6074         /* ACQUIRE_MEM - make one or more surfaces valid for use by the subsequent operations */
6075         amdgpu_ring_write(ring, PACKET3(PACKET3_ACQUIRE_MEM, 6));
6076         amdgpu_ring_write(ring, 0); /* CP_COHER_CNTL */
6077         amdgpu_ring_write(ring, 0xffffffff);  /* CP_COHER_SIZE */
6078         amdgpu_ring_write(ring, 0xffffff);  /* CP_COHER_SIZE_HI */
6079         amdgpu_ring_write(ring, 0); /* CP_COHER_BASE */
6080         amdgpu_ring_write(ring, 0);  /* CP_COHER_BASE_HI */
6081         amdgpu_ring_write(ring, 0x0000000A); /* POLL_INTERVAL */
6082         amdgpu_ring_write(ring, gcr_cntl); /* GCR_CNTL */
6083 }
6084
6085 static const struct amd_ip_funcs gfx_v11_0_ip_funcs = {
6086         .name = "gfx_v11_0",
6087         .early_init = gfx_v11_0_early_init,
6088         .late_init = gfx_v11_0_late_init,
6089         .sw_init = gfx_v11_0_sw_init,
6090         .sw_fini = gfx_v11_0_sw_fini,
6091         .hw_init = gfx_v11_0_hw_init,
6092         .hw_fini = gfx_v11_0_hw_fini,
6093         .suspend = gfx_v11_0_suspend,
6094         .resume = gfx_v11_0_resume,
6095         .is_idle = gfx_v11_0_is_idle,
6096         .wait_for_idle = gfx_v11_0_wait_for_idle,
6097         .soft_reset = gfx_v11_0_soft_reset,
6098         .check_soft_reset = gfx_v11_0_check_soft_reset,
6099         .post_soft_reset = gfx_v11_0_post_soft_reset,
6100         .set_clockgating_state = gfx_v11_0_set_clockgating_state,
6101         .set_powergating_state = gfx_v11_0_set_powergating_state,
6102         .get_clockgating_state = gfx_v11_0_get_clockgating_state,
6103 };
6104
6105 static const struct amdgpu_ring_funcs gfx_v11_0_ring_funcs_gfx = {
6106         .type = AMDGPU_RING_TYPE_GFX,
6107         .align_mask = 0xff,
6108         .nop = PACKET3(PACKET3_NOP, 0x3FFF),
6109         .support_64bit_ptrs = true,
6110         .secure_submission_supported = true,
6111         .get_rptr = gfx_v11_0_ring_get_rptr_gfx,
6112         .get_wptr = gfx_v11_0_ring_get_wptr_gfx,
6113         .set_wptr = gfx_v11_0_ring_set_wptr_gfx,
6114         .emit_frame_size = /* totally 242 maximum if 16 IBs */
6115                 5 + /* COND_EXEC */
6116                 9 + /* SET_Q_PREEMPTION_MODE */
6117                 7 + /* PIPELINE_SYNC */
6118                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
6119                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
6120                 2 + /* VM_FLUSH */
6121                 8 + /* FENCE for VM_FLUSH */
6122                 20 + /* GDS switch */
6123                 5 + /* COND_EXEC */
6124                 7 + /* HDP_flush */
6125                 4 + /* VGT_flush */
6126                 31 + /* DE_META */
6127                 3 + /* CNTX_CTRL */
6128                 5 + /* HDP_INVL */
6129                 8 + 8 + /* FENCE x2 */
6130                 8, /* gfx_v11_0_emit_mem_sync */
6131         .emit_ib_size = 4, /* gfx_v11_0_ring_emit_ib_gfx */
6132         .emit_ib = gfx_v11_0_ring_emit_ib_gfx,
6133         .emit_fence = gfx_v11_0_ring_emit_fence,
6134         .emit_pipeline_sync = gfx_v11_0_ring_emit_pipeline_sync,
6135         .emit_vm_flush = gfx_v11_0_ring_emit_vm_flush,
6136         .emit_gds_switch = gfx_v11_0_ring_emit_gds_switch,
6137         .emit_hdp_flush = gfx_v11_0_ring_emit_hdp_flush,
6138         .test_ring = gfx_v11_0_ring_test_ring,
6139         .test_ib = gfx_v11_0_ring_test_ib,
6140         .insert_nop = amdgpu_ring_insert_nop,
6141         .pad_ib = amdgpu_ring_generic_pad_ib,
6142         .emit_cntxcntl = gfx_v11_0_ring_emit_cntxcntl,
6143         .emit_gfx_shadow = gfx_v11_0_ring_emit_gfx_shadow,
6144         .init_cond_exec = gfx_v11_0_ring_emit_init_cond_exec,
6145         .patch_cond_exec = gfx_v11_0_ring_emit_patch_cond_exec,
6146         .preempt_ib = gfx_v11_0_ring_preempt_ib,
6147         .emit_frame_cntl = gfx_v11_0_ring_emit_frame_cntl,
6148         .emit_wreg = gfx_v11_0_ring_emit_wreg,
6149         .emit_reg_wait = gfx_v11_0_ring_emit_reg_wait,
6150         .emit_reg_write_reg_wait = gfx_v11_0_ring_emit_reg_write_reg_wait,
6151         .soft_recovery = gfx_v11_0_ring_soft_recovery,
6152         .emit_mem_sync = gfx_v11_0_emit_mem_sync,
6153 };
6154
6155 static const struct amdgpu_ring_funcs gfx_v11_0_ring_funcs_compute = {
6156         .type = AMDGPU_RING_TYPE_COMPUTE,
6157         .align_mask = 0xff,
6158         .nop = PACKET3(PACKET3_NOP, 0x3FFF),
6159         .support_64bit_ptrs = true,
6160         .get_rptr = gfx_v11_0_ring_get_rptr_compute,
6161         .get_wptr = gfx_v11_0_ring_get_wptr_compute,
6162         .set_wptr = gfx_v11_0_ring_set_wptr_compute,
6163         .emit_frame_size =
6164                 20 + /* gfx_v11_0_ring_emit_gds_switch */
6165                 7 + /* gfx_v11_0_ring_emit_hdp_flush */
6166                 5 + /* hdp invalidate */
6167                 7 + /* gfx_v11_0_ring_emit_pipeline_sync */
6168                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
6169                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
6170                 2 + /* gfx_v11_0_ring_emit_vm_flush */
6171                 8 + 8 + 8 + /* gfx_v11_0_ring_emit_fence x3 for user fence, vm fence */
6172                 8, /* gfx_v11_0_emit_mem_sync */
6173         .emit_ib_size = 7, /* gfx_v11_0_ring_emit_ib_compute */
6174         .emit_ib = gfx_v11_0_ring_emit_ib_compute,
6175         .emit_fence = gfx_v11_0_ring_emit_fence,
6176         .emit_pipeline_sync = gfx_v11_0_ring_emit_pipeline_sync,
6177         .emit_vm_flush = gfx_v11_0_ring_emit_vm_flush,
6178         .emit_gds_switch = gfx_v11_0_ring_emit_gds_switch,
6179         .emit_hdp_flush = gfx_v11_0_ring_emit_hdp_flush,
6180         .test_ring = gfx_v11_0_ring_test_ring,
6181         .test_ib = gfx_v11_0_ring_test_ib,
6182         .insert_nop = amdgpu_ring_insert_nop,
6183         .pad_ib = amdgpu_ring_generic_pad_ib,
6184         .emit_wreg = gfx_v11_0_ring_emit_wreg,
6185         .emit_reg_wait = gfx_v11_0_ring_emit_reg_wait,
6186         .emit_reg_write_reg_wait = gfx_v11_0_ring_emit_reg_write_reg_wait,
6187         .emit_mem_sync = gfx_v11_0_emit_mem_sync,
6188 };
6189
6190 static const struct amdgpu_ring_funcs gfx_v11_0_ring_funcs_kiq = {
6191         .type = AMDGPU_RING_TYPE_KIQ,
6192         .align_mask = 0xff,
6193         .nop = PACKET3(PACKET3_NOP, 0x3FFF),
6194         .support_64bit_ptrs = true,
6195         .get_rptr = gfx_v11_0_ring_get_rptr_compute,
6196         .get_wptr = gfx_v11_0_ring_get_wptr_compute,
6197         .set_wptr = gfx_v11_0_ring_set_wptr_compute,
6198         .emit_frame_size =
6199                 20 + /* gfx_v11_0_ring_emit_gds_switch */
6200                 7 + /* gfx_v11_0_ring_emit_hdp_flush */
6201                 5 + /*hdp invalidate */
6202                 7 + /* gfx_v11_0_ring_emit_pipeline_sync */
6203                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
6204                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
6205                 2 + /* gfx_v11_0_ring_emit_vm_flush */
6206                 8 + 8 + 8, /* gfx_v11_0_ring_emit_fence_kiq x3 for user fence, vm fence */
6207         .emit_ib_size = 7, /* gfx_v11_0_ring_emit_ib_compute */
6208         .emit_ib = gfx_v11_0_ring_emit_ib_compute,
6209         .emit_fence = gfx_v11_0_ring_emit_fence_kiq,
6210         .test_ring = gfx_v11_0_ring_test_ring,
6211         .test_ib = gfx_v11_0_ring_test_ib,
6212         .insert_nop = amdgpu_ring_insert_nop,
6213         .pad_ib = amdgpu_ring_generic_pad_ib,
6214         .emit_rreg = gfx_v11_0_ring_emit_rreg,
6215         .emit_wreg = gfx_v11_0_ring_emit_wreg,
6216         .emit_reg_wait = gfx_v11_0_ring_emit_reg_wait,
6217         .emit_reg_write_reg_wait = gfx_v11_0_ring_emit_reg_write_reg_wait,
6218 };
6219
6220 static void gfx_v11_0_set_ring_funcs(struct amdgpu_device *adev)
6221 {
6222         int i;
6223
6224         adev->gfx.kiq[0].ring.funcs = &gfx_v11_0_ring_funcs_kiq;
6225
6226         for (i = 0; i < adev->gfx.num_gfx_rings; i++)
6227                 adev->gfx.gfx_ring[i].funcs = &gfx_v11_0_ring_funcs_gfx;
6228
6229         for (i = 0; i < adev->gfx.num_compute_rings; i++)
6230                 adev->gfx.compute_ring[i].funcs = &gfx_v11_0_ring_funcs_compute;
6231 }
6232
6233 static const struct amdgpu_irq_src_funcs gfx_v11_0_eop_irq_funcs = {
6234         .set = gfx_v11_0_set_eop_interrupt_state,
6235         .process = gfx_v11_0_eop_irq,
6236 };
6237
6238 static const struct amdgpu_irq_src_funcs gfx_v11_0_priv_reg_irq_funcs = {
6239         .set = gfx_v11_0_set_priv_reg_fault_state,
6240         .process = gfx_v11_0_priv_reg_irq,
6241 };
6242
6243 static const struct amdgpu_irq_src_funcs gfx_v11_0_priv_inst_irq_funcs = {
6244         .set = gfx_v11_0_set_priv_inst_fault_state,
6245         .process = gfx_v11_0_priv_inst_irq,
6246 };
6247
6248 static const struct amdgpu_irq_src_funcs gfx_v11_0_rlc_gc_fed_irq_funcs = {
6249         .process = gfx_v11_0_rlc_gc_fed_irq,
6250 };
6251
6252 static void gfx_v11_0_set_irq_funcs(struct amdgpu_device *adev)
6253 {
6254         adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST;
6255         adev->gfx.eop_irq.funcs = &gfx_v11_0_eop_irq_funcs;
6256
6257         adev->gfx.priv_reg_irq.num_types = 1;
6258         adev->gfx.priv_reg_irq.funcs = &gfx_v11_0_priv_reg_irq_funcs;
6259
6260         adev->gfx.priv_inst_irq.num_types = 1;
6261         adev->gfx.priv_inst_irq.funcs = &gfx_v11_0_priv_inst_irq_funcs;
6262
6263         adev->gfx.rlc_gc_fed_irq.num_types = 1; /* 0x80 FED error */
6264         adev->gfx.rlc_gc_fed_irq.funcs = &gfx_v11_0_rlc_gc_fed_irq_funcs;
6265
6266 }
6267
6268 static void gfx_v11_0_set_imu_funcs(struct amdgpu_device *adev)
6269 {
6270         if (adev->flags & AMD_IS_APU)
6271                 adev->gfx.imu.mode = MISSION_MODE;
6272         else
6273                 adev->gfx.imu.mode = DEBUG_MODE;
6274
6275         adev->gfx.imu.funcs = &gfx_v11_0_imu_funcs;
6276 }
6277
6278 static void gfx_v11_0_set_rlc_funcs(struct amdgpu_device *adev)
6279 {
6280         adev->gfx.rlc.funcs = &gfx_v11_0_rlc_funcs;
6281 }
6282
6283 static void gfx_v11_0_set_gds_init(struct amdgpu_device *adev)
6284 {
6285         unsigned total_cu = adev->gfx.config.max_cu_per_sh *
6286                             adev->gfx.config.max_sh_per_se *
6287                             adev->gfx.config.max_shader_engines;
6288
6289         adev->gds.gds_size = 0x1000;
6290         adev->gds.gds_compute_max_wave_id = total_cu * 32 - 1;
6291         adev->gds.gws_size = 64;
6292         adev->gds.oa_size = 16;
6293 }
6294
6295 static void gfx_v11_0_set_mqd_funcs(struct amdgpu_device *adev)
6296 {
6297         /* set gfx eng mqd */
6298         adev->mqds[AMDGPU_HW_IP_GFX].mqd_size =
6299                 sizeof(struct v11_gfx_mqd);
6300         adev->mqds[AMDGPU_HW_IP_GFX].init_mqd =
6301                 gfx_v11_0_gfx_mqd_init;
6302         /* set compute eng mqd */
6303         adev->mqds[AMDGPU_HW_IP_COMPUTE].mqd_size =
6304                 sizeof(struct v11_compute_mqd);
6305         adev->mqds[AMDGPU_HW_IP_COMPUTE].init_mqd =
6306                 gfx_v11_0_compute_mqd_init;
6307 }
6308
6309 static void gfx_v11_0_set_user_wgp_inactive_bitmap_per_sh(struct amdgpu_device *adev,
6310                                                           u32 bitmap)
6311 {
6312         u32 data;
6313
6314         if (!bitmap)
6315                 return;
6316
6317         data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_WGPS__SHIFT;
6318         data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_WGPS_MASK;
6319
6320         WREG32_SOC15(GC, 0, regGC_USER_SHADER_ARRAY_CONFIG, data);
6321 }
6322
6323 static u32 gfx_v11_0_get_wgp_active_bitmap_per_sh(struct amdgpu_device *adev)
6324 {
6325         u32 data, wgp_bitmask;
6326         data = RREG32_SOC15(GC, 0, regCC_GC_SHADER_ARRAY_CONFIG);
6327         data |= RREG32_SOC15(GC, 0, regGC_USER_SHADER_ARRAY_CONFIG);
6328
6329         data &= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_WGPS_MASK;
6330         data >>= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_WGPS__SHIFT;
6331
6332         wgp_bitmask =
6333                 amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh >> 1);
6334
6335         return (~data) & wgp_bitmask;
6336 }
6337
6338 static u32 gfx_v11_0_get_cu_active_bitmap_per_sh(struct amdgpu_device *adev)
6339 {
6340         u32 wgp_idx, wgp_active_bitmap;
6341         u32 cu_bitmap_per_wgp, cu_active_bitmap;
6342
6343         wgp_active_bitmap = gfx_v11_0_get_wgp_active_bitmap_per_sh(adev);
6344         cu_active_bitmap = 0;
6345
6346         for (wgp_idx = 0; wgp_idx < 16; wgp_idx++) {
6347                 /* if there is one WGP enabled, it means 2 CUs will be enabled */
6348                 cu_bitmap_per_wgp = 3 << (2 * wgp_idx);
6349                 if (wgp_active_bitmap & (1 << wgp_idx))
6350                         cu_active_bitmap |= cu_bitmap_per_wgp;
6351         }
6352
6353         return cu_active_bitmap;
6354 }
6355
6356 static int gfx_v11_0_get_cu_info(struct amdgpu_device *adev,
6357                                  struct amdgpu_cu_info *cu_info)
6358 {
6359         int i, j, k, counter, active_cu_number = 0;
6360         u32 mask, bitmap;
6361         unsigned disable_masks[8 * 2];
6362
6363         if (!adev || !cu_info)
6364                 return -EINVAL;
6365
6366         amdgpu_gfx_parse_disable_cu(disable_masks, 8, 2);
6367
6368         mutex_lock(&adev->grbm_idx_mutex);
6369         for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
6370                 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
6371                         mask = 1;
6372                         counter = 0;
6373                         gfx_v11_0_select_se_sh(adev, i, j, 0xffffffff, 0);
6374                         if (i < 8 && j < 2)
6375                                 gfx_v11_0_set_user_wgp_inactive_bitmap_per_sh(
6376                                         adev, disable_masks[i * 2 + j]);
6377                         bitmap = gfx_v11_0_get_cu_active_bitmap_per_sh(adev);
6378
6379                         /**
6380                          * GFX11 could support more than 4 SEs, while the bitmap
6381                          * in cu_info struct is 4x4 and ioctl interface struct
6382                          * drm_amdgpu_info_device should keep stable.
6383                          * So we use last two columns of bitmap to store cu mask for
6384                          * SEs 4 to 7, the layout of the bitmap is as below:
6385                          *    SE0: {SH0,SH1} --> {bitmap[0][0], bitmap[0][1]}
6386                          *    SE1: {SH0,SH1} --> {bitmap[1][0], bitmap[1][1]}
6387                          *    SE2: {SH0,SH1} --> {bitmap[2][0], bitmap[2][1]}
6388                          *    SE3: {SH0,SH1} --> {bitmap[3][0], bitmap[3][1]}
6389                          *    SE4: {SH0,SH1} --> {bitmap[0][2], bitmap[0][3]}
6390                          *    SE5: {SH0,SH1} --> {bitmap[1][2], bitmap[1][3]}
6391                          *    SE6: {SH0,SH1} --> {bitmap[2][2], bitmap[2][3]}
6392                          *    SE7: {SH0,SH1} --> {bitmap[3][2], bitmap[3][3]}
6393                          */
6394                         cu_info->bitmap[0][i % 4][j + (i / 4) * 2] = bitmap;
6395
6396                         for (k = 0; k < adev->gfx.config.max_cu_per_sh; k++) {
6397                                 if (bitmap & mask)
6398                                         counter++;
6399
6400                                 mask <<= 1;
6401                         }
6402                         active_cu_number += counter;
6403                 }
6404         }
6405         gfx_v11_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
6406         mutex_unlock(&adev->grbm_idx_mutex);
6407
6408         cu_info->number = active_cu_number;
6409         cu_info->simd_per_cu = NUM_SIMD_PER_CU;
6410
6411         return 0;
6412 }
6413
6414 const struct amdgpu_ip_block_version gfx_v11_0_ip_block =
6415 {
6416         .type = AMD_IP_BLOCK_TYPE_GFX,
6417         .major = 11,
6418         .minor = 0,
6419         .rev = 0,
6420         .funcs = &gfx_v11_0_ip_funcs,
6421 };