drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c

   1 /*
   2  * Copyright 2021 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23 #include <linux/delay.h>
  24 #include <linux/kernel.h>
  25 #include <linux/firmware.h>
  26 #include <linux/module.h>
  27 #include <linux/pci.h>
  28 #include "amdgpu.h"
  29 #include "amdgpu_gfx.h"
  30 #include "amdgpu_psp.h"
  31 #include "amdgpu_smu.h"
  32 #include "amdgpu_atomfirmware.h"
  33 #include "imu_v11_0.h"
  34 #include "soc21.h"
  35 #include "nvd.h"
  36
  37 #include "gc/gc_11_0_0_offset.h"
  38 #include "gc/gc_11_0_0_sh_mask.h"
  39 #include "smuio/smuio_13_0_6_offset.h"
  40 #include "smuio/smuio_13_0_6_sh_mask.h"
  41 #include "navi10_enum.h"
  42 #include "ivsrcid/gfx/irqsrcs_gfx_11_0_0.h"
  43
  44 #include "soc15.h"
  45 #include "soc15d.h"
  46 #include "clearstate_gfx11.h"
  47 #include "v11_structs.h"
  48 #include "gfx_v11_0.h"
  49 #include "gfx_v11_0_3.h"
  50 #include "nbio_v4_3.h"
  51 #include "mes_v11_0.h"
  52
  53 #define GFX11_NUM_GFX_RINGS             1
  54 #define GFX11_MEC_HPD_SIZE      2048
  55
  56 #define RLCG_UCODE_LOADING_START_ADDRESS        0x00002000L
  57 #define RLC_PG_DELAY_3_DEFAULT_GC_11_0_1        0x1388
  58
  59 #define regCGTT_WD_CLK_CTRL             0x5086
  60 #define regCGTT_WD_CLK_CTRL_BASE_IDX    1
  61 #define regRLC_RLCS_BOOTLOAD_STATUS_gc_11_0_1   0x4e7e
  62 #define regRLC_RLCS_BOOTLOAD_STATUS_gc_11_0_1_BASE_IDX  1
  63 #define regPC_CONFIG_CNTL_1             0x194d
  64 #define regPC_CONFIG_CNTL_1_BASE_IDX    1
  65
  66 MODULE_FIRMWARE("amdgpu/gc_11_0_0_pfp.bin");
  67 MODULE_FIRMWARE("amdgpu/gc_11_0_0_me.bin");
  68 MODULE_FIRMWARE("amdgpu/gc_11_0_0_mec.bin");
  69 MODULE_FIRMWARE("amdgpu/gc_11_0_0_rlc.bin");
  70 MODULE_FIRMWARE("amdgpu/gc_11_0_0_rlc_1.bin");
  71 MODULE_FIRMWARE("amdgpu/gc_11_0_0_toc.bin");
  72 MODULE_FIRMWARE("amdgpu/gc_11_0_1_pfp.bin");
  73 MODULE_FIRMWARE("amdgpu/gc_11_0_1_me.bin");
  74 MODULE_FIRMWARE("amdgpu/gc_11_0_1_mec.bin");
  75 MODULE_FIRMWARE("amdgpu/gc_11_0_1_rlc.bin");
  76 MODULE_FIRMWARE("amdgpu/gc_11_0_2_pfp.bin");
  77 MODULE_FIRMWARE("amdgpu/gc_11_0_2_me.bin");
  78 MODULE_FIRMWARE("amdgpu/gc_11_0_2_mec.bin");
  79 MODULE_FIRMWARE("amdgpu/gc_11_0_2_rlc.bin");
  80 MODULE_FIRMWARE("amdgpu/gc_11_0_3_pfp.bin");
  81 MODULE_FIRMWARE("amdgpu/gc_11_0_3_me.bin");
  82 MODULE_FIRMWARE("amdgpu/gc_11_0_3_mec.bin");
  83 MODULE_FIRMWARE("amdgpu/gc_11_0_3_rlc.bin");
  84 MODULE_FIRMWARE("amdgpu/gc_11_0_4_pfp.bin");
  85 MODULE_FIRMWARE("amdgpu/gc_11_0_4_me.bin");
  86 MODULE_FIRMWARE("amdgpu/gc_11_0_4_mec.bin");
  87 MODULE_FIRMWARE("amdgpu/gc_11_0_4_rlc.bin");
  88 MODULE_FIRMWARE("amdgpu/gc_11_5_0_pfp.bin");
  89 MODULE_FIRMWARE("amdgpu/gc_11_5_0_me.bin");
  90 MODULE_FIRMWARE("amdgpu/gc_11_5_0_mec.bin");
  91 MODULE_FIRMWARE("amdgpu/gc_11_5_0_rlc.bin");
  92 MODULE_FIRMWARE("amdgpu/gc_11_5_1_pfp.bin");
  93 MODULE_FIRMWARE("amdgpu/gc_11_5_1_me.bin");
  94 MODULE_FIRMWARE("amdgpu/gc_11_5_1_mec.bin");
  95 MODULE_FIRMWARE("amdgpu/gc_11_5_1_rlc.bin");
  96 MODULE_FIRMWARE("amdgpu/gc_11_5_2_pfp.bin");
  97 MODULE_FIRMWARE("amdgpu/gc_11_5_2_me.bin");
  98 MODULE_FIRMWARE("amdgpu/gc_11_5_2_mec.bin");
  99 MODULE_FIRMWARE("amdgpu/gc_11_5_2_rlc.bin");
 100
 101 static const struct amdgpu_hwip_reg_entry gc_reg_list_11_0[] = {
 102         SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS),
 103         SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS2),
 104         SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS3),
 105         SOC15_REG_ENTRY_STR(GC, 0, regCP_STALLED_STAT1),
 106         SOC15_REG_ENTRY_STR(GC, 0, regCP_STALLED_STAT2),
 107         SOC15_REG_ENTRY_STR(GC, 0, regCP_STALLED_STAT3),
 108         SOC15_REG_ENTRY_STR(GC, 0, regCP_CPC_STALLED_STAT1),
 109         SOC15_REG_ENTRY_STR(GC, 0, regCP_CPF_STALLED_STAT1),
 110         SOC15_REG_ENTRY_STR(GC, 0, regCP_BUSY_STAT),
 111         SOC15_REG_ENTRY_STR(GC, 0, regCP_CPC_BUSY_STAT),
 112         SOC15_REG_ENTRY_STR(GC, 0, regCP_CPF_BUSY_STAT),
 113         SOC15_REG_ENTRY_STR(GC, 0, regCP_CPC_BUSY_STAT2),
 114         SOC15_REG_ENTRY_STR(GC, 0, regCP_CPF_BUSY_STAT2),
 115         SOC15_REG_ENTRY_STR(GC, 0, regCP_CPF_STATUS),
 116         SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_ERROR),
 117         SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HPD_STATUS0),
 118         SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_BASE),
 119         SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_RPTR),
 120         SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_WPTR),
 121         SOC15_REG_ENTRY_STR(GC, 0, regCP_RB0_BASE),
 122         SOC15_REG_ENTRY_STR(GC, 0, regCP_RB0_RPTR),
 123         SOC15_REG_ENTRY_STR(GC, 0, regCP_RB0_WPTR),
 124         SOC15_REG_ENTRY_STR(GC, 0, regCP_RB1_BASE),
 125         SOC15_REG_ENTRY_STR(GC, 0, regCP_RB1_RPTR),
 126         SOC15_REG_ENTRY_STR(GC, 0, regCP_RB1_WPTR),
 127         SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_CMD_BUFSZ),
 128         SOC15_REG_ENTRY_STR(GC, 0, regCP_IB2_CMD_BUFSZ),
 129         SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BASE_LO),
 130         SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BASE_HI),
 131         SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BUFSZ),
 132         SOC15_REG_ENTRY_STR(GC, 0, regCP_IB2_BASE_LO),
 133         SOC15_REG_ENTRY_STR(GC, 0, regCP_IB2_BASE_HI),
 134         SOC15_REG_ENTRY_STR(GC, 0, regCP_IB2_BUFSZ),
 135         SOC15_REG_ENTRY_STR(GC, 0, regCPF_UTCL1_STATUS),
 136         SOC15_REG_ENTRY_STR(GC, 0, regCPC_UTCL1_STATUS),
 137         SOC15_REG_ENTRY_STR(GC, 0, regCPG_UTCL1_STATUS),
 138         SOC15_REG_ENTRY_STR(GC, 0, regGDS_PROTECTION_FAULT),
 139         SOC15_REG_ENTRY_STR(GC, 0, regGDS_VM_PROTECTION_FAULT),
 140         SOC15_REG_ENTRY_STR(GC, 0, regIA_UTCL1_STATUS),
 141         SOC15_REG_ENTRY_STR(GC, 0, regIA_UTCL1_STATUS_2),
 142         SOC15_REG_ENTRY_STR(GC, 0, regPA_CL_CNTL_STATUS),
 143         SOC15_REG_ENTRY_STR(GC, 0, regRLC_UTCL1_STATUS),
 144         SOC15_REG_ENTRY_STR(GC, 0, regRMI_UTCL1_STATUS),
 145         SOC15_REG_ENTRY_STR(GC, 0, regSQC_CACHES),
 146         SOC15_REG_ENTRY_STR(GC, 0, regSQG_STATUS),
 147         SOC15_REG_ENTRY_STR(GC, 0, regWD_UTCL1_STATUS),
 148         SOC15_REG_ENTRY_STR(GC, 0, regGCVM_L2_PROTECTION_FAULT_CNTL),
 149         SOC15_REG_ENTRY_STR(GC, 0, regGCVM_L2_PROTECTION_FAULT_STATUS),
 150         SOC15_REG_ENTRY_STR(GC, 0, regCP_DEBUG),
 151         SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_CNTL),
 152         SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_CNTL),
 153         SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC1_INSTR_PNTR),
 154         SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_DEBUG_INTERRUPT_INSTR_PNTR),
 155         SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_INSTR_PNTR),
 156         SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_INSTR_PNTR),
 157         SOC15_REG_ENTRY_STR(GC, 0, regCP_PFP_INSTR_PNTR),
 158         SOC15_REG_ENTRY_STR(GC, 0, regCP_CPC_STATUS),
 159         /* cp header registers */
 160         SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_ME1_HEADER_DUMP),
 161         SOC15_REG_ENTRY_STR(GC, 0, regCP_PFP_HEADER_DUMP),
 162         SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP),
 163         SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP),
 164         /* SE status registers */
 165         SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE0),
 166         SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE1),
 167         SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE2),
 168         SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE3),
 169         SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE4),
 170         SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE5)
 171 };
 172
 173 static const struct amdgpu_hwip_reg_entry gc_cp_reg_list_11[] = {
 174         /* compute registers */
 175         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_VMID),
 176         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PERSISTENT_STATE),
 177         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PIPE_PRIORITY),
 178         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_QUEUE_PRIORITY),
 179         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_QUANTUM),
 180         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_BASE),
 181         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_BASE_HI),
 182         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_RPTR),
 183         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR),
 184         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR_HI),
 185         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL),
 186         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_CONTROL),
 187         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_IB_BASE_ADDR),
 188         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_IB_BASE_ADDR_HI),
 189         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_IB_RPTR),
 190         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_IB_CONTROL),
 191         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_DEQUEUE_REQUEST),
 192         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_BASE_ADDR),
 193         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_BASE_ADDR_HI),
 194         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_CONTROL),
 195         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_RPTR),
 196         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_WPTR),
 197         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_EVENTS),
 198         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CTX_SAVE_BASE_ADDR_LO),
 199         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CTX_SAVE_BASE_ADDR_HI),
 200         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CTX_SAVE_CONTROL),
 201         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CNTL_STACK_OFFSET),
 202         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CNTL_STACK_SIZE),
 203         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_WG_STATE_OFFSET),
 204         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CTX_SAVE_SIZE),
 205         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_GDS_RESOURCE_STATE),
 206         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_ERROR),
 207         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_WPTR_MEM),
 208         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_WPTR_LO),
 209         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_WPTR_HI),
 210         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_SUSPEND_CNTL_STACK_OFFSET),
 211         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_SUSPEND_CNTL_STACK_DW_CNT),
 212         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_SUSPEND_WG_STATE_OFFSET),
 213         SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_DEQUEUE_STATUS)
 214 };
 215
 216 static const struct amdgpu_hwip_reg_entry gc_gfx_queue_reg_list_11[] = {
 217         /* gfx queue registers */
 218         SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_ACTIVE),
 219         SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_VMID),
 220         SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_QUEUE_PRIORITY),
 221         SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_QUANTUM),
 222         SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_BASE),
 223         SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_BASE_HI),
 224         SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_OFFSET),
 225         SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_CNTL),
 226         SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_CSMD_RPTR),
 227         SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_WPTR),
 228         SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_WPTR_HI),
 229         SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_DEQUEUE_REQUEST),
 230         SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_MAPPED),
 231         SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_QUE_MGR_CONTROL),
 232         SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_HQ_CONTROL0),
 233         SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_HQ_STATUS0),
 234         SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_MQD_BASE_ADDR),
 235         SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_MQD_BASE_ADDR_HI),
 236         SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_WPTR_POLL_ADDR_LO),
 237         SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_WPTR_POLL_ADDR_HI),
 238         SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_RPTR),
 239         SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BASE_LO),
 240         SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BASE_HI),
 241         SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_CMD_BUFSZ),
 242         SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BUFSZ)
 243 };
 244
 245 static const struct soc15_reg_golden golden_settings_gc_11_0[] = {
 246         SOC15_REG_GOLDEN_VALUE(GC, 0, regTCP_CNTL, 0x20000000, 0x20000000)
 247 };
 248
 249 static const struct soc15_reg_golden golden_settings_gc_11_0_1[] =
 250 {
 251         SOC15_REG_GOLDEN_VALUE(GC, 0, regCGTT_GS_NGG_CLK_CTRL, 0x9fff8fff, 0x00000010),
 252         SOC15_REG_GOLDEN_VALUE(GC, 0, regCGTT_WD_CLK_CTRL, 0xffff8fff, 0x00000010),
 253         SOC15_REG_GOLDEN_VALUE(GC, 0, regCPF_GCR_CNTL, 0x0007ffff, 0x0000c200),
 254         SOC15_REG_GOLDEN_VALUE(GC, 0, regGL2C_CTRL3, 0xffff001b, 0x00f01988),
 255         SOC15_REG_GOLDEN_VALUE(GC, 0, regPA_CL_ENHANCE, 0xf0ffffff, 0x00880007),
 256         SOC15_REG_GOLDEN_VALUE(GC, 0, regPA_SC_ENHANCE_3, 0xfffffffd, 0x00000008),
 257         SOC15_REG_GOLDEN_VALUE(GC, 0, regPA_SC_VRS_SURFACE_CNTL_1, 0xfff891ff, 0x55480100),
 258         SOC15_REG_GOLDEN_VALUE(GC, 0, regTA_CNTL_AUX, 0xf7f7ffff, 0x01030000),
 259         SOC15_REG_GOLDEN_VALUE(GC, 0, regTCP_CNTL2, 0xfcffffff, 0x0000000a)
 260 };
 261
 262 #define DEFAULT_SH_MEM_CONFIG \
 263         ((SH_MEM_ADDRESS_MODE_64 << SH_MEM_CONFIG__ADDRESS_MODE__SHIFT) | \
 264          (SH_MEM_ALIGNMENT_MODE_UNALIGNED << SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT) | \
 265          (3 << SH_MEM_CONFIG__INITIAL_INST_PREFETCH__SHIFT))
 266
 267 static void gfx_v11_0_disable_gpa_mode(struct amdgpu_device *adev);
 268 static void gfx_v11_0_set_ring_funcs(struct amdgpu_device *adev);
 269 static void gfx_v11_0_set_irq_funcs(struct amdgpu_device *adev);
 270 static void gfx_v11_0_set_gds_init(struct amdgpu_device *adev);
 271 static void gfx_v11_0_set_rlc_funcs(struct amdgpu_device *adev);
 272 static void gfx_v11_0_set_mqd_funcs(struct amdgpu_device *adev);
 273 static void gfx_v11_0_set_imu_funcs(struct amdgpu_device *adev);
 274 static int gfx_v11_0_get_cu_info(struct amdgpu_device *adev,
 275                                  struct amdgpu_cu_info *cu_info);
 276 static uint64_t gfx_v11_0_get_gpu_clock_counter(struct amdgpu_device *adev);
 277 static void gfx_v11_0_select_se_sh(struct amdgpu_device *adev, u32 se_num,
 278                                    u32 sh_num, u32 instance, int xcc_id);
 279 static u32 gfx_v11_0_get_wgp_active_bitmap_per_sh(struct amdgpu_device *adev);
 280
 281 static void gfx_v11_0_ring_emit_de_meta(struct amdgpu_ring *ring, bool resume);
 282 static void gfx_v11_0_ring_emit_frame_cntl(struct amdgpu_ring *ring, bool start, bool secure);
 283 static void gfx_v11_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg,
 284                                      uint32_t val);
 285 static int gfx_v11_0_wait_for_rlc_autoload_complete(struct amdgpu_device *adev);
 286 static void gfx_v11_0_ring_invalidate_tlbs(struct amdgpu_ring *ring,
 287                                            uint16_t pasid, uint32_t flush_type,
 288                                            bool all_hub, uint8_t dst_sel);
 289 static void gfx_v11_0_set_safe_mode(struct amdgpu_device *adev, int xcc_id);
 290 static void gfx_v11_0_unset_safe_mode(struct amdgpu_device *adev, int xcc_id);
 291 static void gfx_v11_0_update_perf_clk(struct amdgpu_device *adev,
 292                                       bool enable);
 293
 294 static void gfx11_kiq_set_resources(struct amdgpu_ring *kiq_ring, uint64_t queue_mask)
 295 {
 296         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_RESOURCES, 6));
 297         amdgpu_ring_write(kiq_ring, PACKET3_SET_RESOURCES_VMID_MASK(0) |
 298                           PACKET3_SET_RESOURCES_UNMAP_LATENTY(0xa) | /* unmap_latency: 0xa (~ 1s) */
 299                           PACKET3_SET_RESOURCES_QUEUE_TYPE(0)); /* vmid_mask:0 queue_type:0 (KIQ) */
 300         amdgpu_ring_write(kiq_ring, lower_32_bits(queue_mask)); /* queue mask lo */
 301         amdgpu_ring_write(kiq_ring, upper_32_bits(queue_mask)); /* queue mask hi */
 302         amdgpu_ring_write(kiq_ring, 0); /* gws mask lo */
 303         amdgpu_ring_write(kiq_ring, 0); /* gws mask hi */
 304         amdgpu_ring_write(kiq_ring, 0); /* oac mask */
 305         amdgpu_ring_write(kiq_ring, 0); /* gds heap base:0, gds heap size:0 */
 306 }
 307
 308 static void gfx11_kiq_map_queues(struct amdgpu_ring *kiq_ring,
 309                                  struct amdgpu_ring *ring)
 310 {
 311         uint64_t mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj);
 312         uint64_t wptr_addr = ring->wptr_gpu_addr;
 313         uint32_t me = 0, eng_sel = 0;
 314
 315         switch (ring->funcs->type) {
 316         case AMDGPU_RING_TYPE_COMPUTE:
 317                 me = 1;
 318                 eng_sel = 0;
 319                 break;
 320         case AMDGPU_RING_TYPE_GFX:
 321                 me = 0;
 322                 eng_sel = 4;
 323                 break;
 324         case AMDGPU_RING_TYPE_MES:
 325                 me = 2;
 326                 eng_sel = 5;
 327                 break;
 328         default:
 329                 WARN_ON(1);
 330         }
 331
 332         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5));
 333         /* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/
 334         amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
 335                           PACKET3_MAP_QUEUES_QUEUE_SEL(0) | /* Queue_Sel */
 336                           PACKET3_MAP_QUEUES_VMID(0) | /* VMID */
 337                           PACKET3_MAP_QUEUES_QUEUE(ring->queue) |
 338                           PACKET3_MAP_QUEUES_PIPE(ring->pipe) |
 339                           PACKET3_MAP_QUEUES_ME((me)) |
 340                           PACKET3_MAP_QUEUES_QUEUE_TYPE(0) | /*queue_type: normal compute queue */
 341                           PACKET3_MAP_QUEUES_ALLOC_FORMAT(0) | /* alloc format: all_on_one_pipe */
 342                           PACKET3_MAP_QUEUES_ENGINE_SEL(eng_sel) |
 343                           PACKET3_MAP_QUEUES_NUM_QUEUES(1)); /* num_queues: must be 1 */
 344         amdgpu_ring_write(kiq_ring, PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index));
 345         amdgpu_ring_write(kiq_ring, lower_32_bits(mqd_addr));
 346         amdgpu_ring_write(kiq_ring, upper_32_bits(mqd_addr));
 347         amdgpu_ring_write(kiq_ring, lower_32_bits(wptr_addr));
 348         amdgpu_ring_write(kiq_ring, upper_32_bits(wptr_addr));
 349 }
 350
 351 static void gfx11_kiq_unmap_queues(struct amdgpu_ring *kiq_ring,
 352                                    struct amdgpu_ring *ring,
 353                                    enum amdgpu_unmap_queues_action action,
 354                                    u64 gpu_addr, u64 seq)
 355 {
 356         struct amdgpu_device *adev = kiq_ring->adev;
 357         uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
 358
 359         if (adev->enable_mes && !adev->gfx.kiq[0].ring.sched.ready) {
 360                 amdgpu_mes_unmap_legacy_queue(adev, ring, action, gpu_addr, seq);
 361                 return;
 362         }
 363
 364         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4));
 365         amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
 366                           PACKET3_UNMAP_QUEUES_ACTION(action) |
 367                           PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) |
 368                           PACKET3_UNMAP_QUEUES_ENGINE_SEL(eng_sel) |
 369                           PACKET3_UNMAP_QUEUES_NUM_QUEUES(1));
 370         amdgpu_ring_write(kiq_ring,
 371                   PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index));
 372
 373         if (action == PREEMPT_QUEUES_NO_UNMAP) {
 374                 amdgpu_ring_write(kiq_ring, lower_32_bits(gpu_addr));
 375                 amdgpu_ring_write(kiq_ring, upper_32_bits(gpu_addr));
 376                 amdgpu_ring_write(kiq_ring, seq);
 377         } else {
 378                 amdgpu_ring_write(kiq_ring, 0);
 379                 amdgpu_ring_write(kiq_ring, 0);
 380                 amdgpu_ring_write(kiq_ring, 0);
 381         }
 382 }
 383
 384 static void gfx11_kiq_query_status(struct amdgpu_ring *kiq_ring,
 385                                    struct amdgpu_ring *ring,
 386                                    u64 addr,
 387                                    u64 seq)
 388 {
 389         uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
 390
 391         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_QUERY_STATUS, 5));
 392         amdgpu_ring_write(kiq_ring,
 393                           PACKET3_QUERY_STATUS_CONTEXT_ID(0) |
 394                           PACKET3_QUERY_STATUS_INTERRUPT_SEL(0) |
 395                           PACKET3_QUERY_STATUS_COMMAND(2));
 396         amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
 397                           PACKET3_QUERY_STATUS_DOORBELL_OFFSET(ring->doorbell_index) |
 398                           PACKET3_QUERY_STATUS_ENG_SEL(eng_sel));
 399         amdgpu_ring_write(kiq_ring, lower_32_bits(addr));
 400         amdgpu_ring_write(kiq_ring, upper_32_bits(addr));
 401         amdgpu_ring_write(kiq_ring, lower_32_bits(seq));
 402         amdgpu_ring_write(kiq_ring, upper_32_bits(seq));
 403 }
 404
 405 static void gfx11_kiq_invalidate_tlbs(struct amdgpu_ring *kiq_ring,
 406                                 uint16_t pasid, uint32_t flush_type,
 407                                 bool all_hub)
 408 {
 409         gfx_v11_0_ring_invalidate_tlbs(kiq_ring, pasid, flush_type, all_hub, 1);
 410 }
 411
 412 static const struct kiq_pm4_funcs gfx_v11_0_kiq_pm4_funcs = {
 413         .kiq_set_resources = gfx11_kiq_set_resources,
 414         .kiq_map_queues = gfx11_kiq_map_queues,
 415         .kiq_unmap_queues = gfx11_kiq_unmap_queues,
 416         .kiq_query_status = gfx11_kiq_query_status,
 417         .kiq_invalidate_tlbs = gfx11_kiq_invalidate_tlbs,
 418         .set_resources_size = 8,
 419         .map_queues_size = 7,
 420         .unmap_queues_size = 6,
 421         .query_status_size = 7,
 422         .invalidate_tlbs_size = 2,
 423 };
 424
 425 static void gfx_v11_0_set_kiq_pm4_funcs(struct amdgpu_device *adev)
 426 {
 427         adev->gfx.kiq[0].pmf = &gfx_v11_0_kiq_pm4_funcs;
 428 }
 429
 430 static void gfx_v11_0_init_golden_registers(struct amdgpu_device *adev)
 431 {
 432         if (amdgpu_sriov_vf(adev))
 433                 return;
 434
 435         switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
 436         case IP_VERSION(11, 0, 1):
 437         case IP_VERSION(11, 0, 4):
 438                 soc15_program_register_sequence(adev,
 439                                                 golden_settings_gc_11_0_1,
 440                                                 (const u32)ARRAY_SIZE(golden_settings_gc_11_0_1));
 441                 break;
 442         default:
 443                 break;
 444         }
 445         soc15_program_register_sequence(adev,
 446                                         golden_settings_gc_11_0,
 447                                         (const u32)ARRAY_SIZE(golden_settings_gc_11_0));
 448
 449 }
 450
 451 static void gfx_v11_0_write_data_to_reg(struct amdgpu_ring *ring, int eng_sel,
 452                                        bool wc, uint32_t reg, uint32_t val)
 453 {
 454         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
 455         amdgpu_ring_write(ring, WRITE_DATA_ENGINE_SEL(eng_sel) |
 456                           WRITE_DATA_DST_SEL(0) | (wc ? WR_CONFIRM : 0));
 457         amdgpu_ring_write(ring, reg);
 458         amdgpu_ring_write(ring, 0);
 459         amdgpu_ring_write(ring, val);
 460 }
 461
 462 static void gfx_v11_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel,
 463                                   int mem_space, int opt, uint32_t addr0,
 464                                   uint32_t addr1, uint32_t ref, uint32_t mask,
 465                                   uint32_t inv)
 466 {
 467         amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
 468         amdgpu_ring_write(ring,
 469                           /* memory (1) or register (0) */
 470                           (WAIT_REG_MEM_MEM_SPACE(mem_space) |
 471                            WAIT_REG_MEM_OPERATION(opt) | /* wait */
 472                            WAIT_REG_MEM_FUNCTION(3) |  /* equal */
 473                            WAIT_REG_MEM_ENGINE(eng_sel)));
 474
 475         if (mem_space)
 476                 BUG_ON(addr0 & 0x3); /* Dword align */
 477         amdgpu_ring_write(ring, addr0);
 478         amdgpu_ring_write(ring, addr1);
 479         amdgpu_ring_write(ring, ref);
 480         amdgpu_ring_write(ring, mask);
 481         amdgpu_ring_write(ring, inv); /* poll interval */
 482 }
 483
 484 static int gfx_v11_0_ring_test_ring(struct amdgpu_ring *ring)
 485 {
 486         struct amdgpu_device *adev = ring->adev;
 487         uint32_t scratch = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG0);
 488         uint32_t tmp = 0;
 489         unsigned i;
 490         int r;
 491
 492         WREG32(scratch, 0xCAFEDEAD);
 493         r = amdgpu_ring_alloc(ring, 5);
 494         if (r) {
 495                 DRM_ERROR("amdgpu: cp failed to lock ring %d (%d).\n",
 496                           ring->idx, r);
 497                 return r;
 498         }
 499
 500         if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ) {
 501                 gfx_v11_0_ring_emit_wreg(ring, scratch, 0xDEADBEEF);
 502         } else {
 503                 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
 504                 amdgpu_ring_write(ring, scratch -
 505                                   PACKET3_SET_UCONFIG_REG_START);
 506                 amdgpu_ring_write(ring, 0xDEADBEEF);
 507         }
 508         amdgpu_ring_commit(ring);
 509
 510         for (i = 0; i < adev->usec_timeout; i++) {
 511                 tmp = RREG32(scratch);
 512                 if (tmp == 0xDEADBEEF)
 513                         break;
 514                 if (amdgpu_emu_mode == 1)
 515                         msleep(1);
 516                 else
 517                         udelay(1);
 518         }
 519
 520         if (i >= adev->usec_timeout)
 521                 r = -ETIMEDOUT;
 522         return r;
 523 }
 524
 525 static int gfx_v11_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 526 {
 527         struct amdgpu_device *adev = ring->adev;
 528         struct amdgpu_ib ib;
 529         struct dma_fence *f = NULL;
 530         unsigned index;
 531         uint64_t gpu_addr;
 532         volatile uint32_t *cpu_ptr;
 533         long r;
 534
 535         /* MES KIQ fw hasn't indirect buffer support for now */
 536         if (adev->enable_mes_kiq &&
 537             ring->funcs->type == AMDGPU_RING_TYPE_KIQ)
 538                 return 0;
 539
 540         memset(&ib, 0, sizeof(ib));
 541
 542         if (ring->is_mes_queue) {
 543                 uint32_t padding, offset;
 544
 545                 offset = amdgpu_mes_ctx_get_offs(ring, AMDGPU_MES_CTX_IB_OFFS);
 546                 padding = amdgpu_mes_ctx_get_offs(ring,
 547                                                   AMDGPU_MES_CTX_PADDING_OFFS);
 548
 549                 ib.gpu_addr = amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset);
 550                 ib.ptr = amdgpu_mes_ctx_get_offs_cpu_addr(ring, offset);
 551
 552                 gpu_addr = amdgpu_mes_ctx_get_offs_gpu_addr(ring, padding);
 553                 cpu_ptr = amdgpu_mes_ctx_get_offs_cpu_addr(ring, padding);
 554                 *cpu_ptr = cpu_to_le32(0xCAFEDEAD);
 555         } else {
 556                 r = amdgpu_device_wb_get(adev, &index);
 557                 if (r)
 558                         return r;
 559
 560                 gpu_addr = adev->wb.gpu_addr + (index * 4);
 561                 adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD);
 562                 cpu_ptr = &adev->wb.wb[index];
 563
 564                 r = amdgpu_ib_get(adev, NULL, 20, AMDGPU_IB_POOL_DIRECT, &ib);
 565                 if (r) {
 566                         DRM_ERROR("amdgpu: failed to get ib (%ld).\n", r);
 567                         goto err1;
 568                 }
 569         }
 570
 571         ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3);
 572         ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM;
 573         ib.ptr[2] = lower_32_bits(gpu_addr);
 574         ib.ptr[3] = upper_32_bits(gpu_addr);
 575         ib.ptr[4] = 0xDEADBEEF;
 576         ib.length_dw = 5;
 577
 578         r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
 579         if (r)
 580                 goto err2;
 581
 582         r = dma_fence_wait_timeout(f, false, timeout);
 583         if (r == 0) {
 584                 r = -ETIMEDOUT;
 585                 goto err2;
 586         } else if (r < 0) {
 587                 goto err2;
 588         }
 589
 590         if (le32_to_cpu(*cpu_ptr) == 0xDEADBEEF)
 591                 r = 0;
 592         else
 593                 r = -EINVAL;
 594 err2:
 595         if (!ring->is_mes_queue)
 596                 amdgpu_ib_free(adev, &ib, NULL);
 597         dma_fence_put(f);
 598 err1:
 599         if (!ring->is_mes_queue)
 600                 amdgpu_device_wb_free(adev, index);
 601         return r;
 602 }
 603
 604 static void gfx_v11_0_free_microcode(struct amdgpu_device *adev)
 605 {
 606         amdgpu_ucode_release(&adev->gfx.pfp_fw);
 607         amdgpu_ucode_release(&adev->gfx.me_fw);
 608         amdgpu_ucode_release(&adev->gfx.rlc_fw);
 609         amdgpu_ucode_release(&adev->gfx.mec_fw);
 610
 611         kfree(adev->gfx.rlc.register_list_format);
 612 }
 613
 614 static int gfx_v11_0_init_toc_microcode(struct amdgpu_device *adev, const char *ucode_prefix)
 615 {
 616         const struct psp_firmware_header_v1_0 *toc_hdr;
 617         int err = 0;
 618
 619         err = amdgpu_ucode_request(adev, &adev->psp.toc_fw,
 620                                    "amdgpu/%s_toc.bin", ucode_prefix);
 621         if (err)
 622                 goto out;
 623
 624         toc_hdr = (const struct psp_firmware_header_v1_0 *)adev->psp.toc_fw->data;
 625         adev->psp.toc.fw_version = le32_to_cpu(toc_hdr->header.ucode_version);
 626         adev->psp.toc.feature_version = le32_to_cpu(toc_hdr->sos.fw_version);
 627         adev->psp.toc.size_bytes = le32_to_cpu(toc_hdr->header.ucode_size_bytes);
 628         adev->psp.toc.start_addr = (uint8_t *)toc_hdr +
 629                                 le32_to_cpu(toc_hdr->header.ucode_array_offset_bytes);
 630         return 0;
 631 out:
 632         amdgpu_ucode_release(&adev->psp.toc_fw);
 633         return err;
 634 }
 635
 636 static void gfx_v11_0_check_fw_cp_gfx_shadow(struct amdgpu_device *adev)
 637 {
 638         switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
 639         case IP_VERSION(11, 0, 0):
 640         case IP_VERSION(11, 0, 2):
 641         case IP_VERSION(11, 0, 3):
 642                 if ((adev->gfx.me_fw_version >= 1505) &&
 643                     (adev->gfx.pfp_fw_version >= 1600) &&
 644                     (adev->gfx.mec_fw_version >= 512)) {
 645                         if (amdgpu_sriov_vf(adev))
 646                                 adev->gfx.cp_gfx_shadow = true;
 647                         else
 648                                 adev->gfx.cp_gfx_shadow = false;
 649                 }
 650                 break;
 651         default:
 652                 adev->gfx.cp_gfx_shadow = false;
 653                 break;
 654         }
 655 }
 656
 657 static int gfx_v11_0_init_microcode(struct amdgpu_device *adev)
 658 {
 659         char ucode_prefix[25];
 660         int err;
 661         const struct rlc_firmware_header_v2_0 *rlc_hdr;
 662         uint16_t version_major;
 663         uint16_t version_minor;
 664
 665         DRM_DEBUG("\n");
 666
 667         amdgpu_ucode_ip_version_decode(adev, GC_HWIP, ucode_prefix, sizeof(ucode_prefix));
 668         err = amdgpu_ucode_request(adev, &adev->gfx.pfp_fw,
 669                                    "amdgpu/%s_pfp.bin", ucode_prefix);
 670         if (err)
 671                 goto out;
 672         /* check pfp fw hdr version to decide if enable rs64 for gfx11.*/
 673         adev->gfx.rs64_enable = amdgpu_ucode_hdr_version(
 674                                 (union amdgpu_firmware_header *)
 675                                 adev->gfx.pfp_fw->data, 2, 0);
 676         if (adev->gfx.rs64_enable) {
 677                 dev_info(adev->dev, "CP RS64 enable\n");
 678                 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_PFP);
 679                 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_PFP_P0_STACK);
 680                 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_PFP_P1_STACK);
 681         } else {
 682                 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_PFP);
 683         }
 684
 685         err = amdgpu_ucode_request(adev, &adev->gfx.me_fw,
 686                                    "amdgpu/%s_me.bin", ucode_prefix);
 687         if (err)
 688                 goto out;
 689         if (adev->gfx.rs64_enable) {
 690                 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_ME);
 691                 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_ME_P0_STACK);
 692                 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_ME_P1_STACK);
 693         } else {
 694                 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_ME);
 695         }
 696
 697         if (!amdgpu_sriov_vf(adev)) {
 698                 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 0, 0) &&
 699                     adev->pdev->revision == 0xCE)
 700                         err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw,
 701                                                    "amdgpu/gc_11_0_0_rlc_1.bin");
 702                 else
 703                         err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw,
 704                                                    "amdgpu/%s_rlc.bin", ucode_prefix);
 705                 if (err)
 706                         goto out;
 707                 rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
 708                 version_major = le16_to_cpu(rlc_hdr->header.header_version_major);
 709                 version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor);
 710                 err = amdgpu_gfx_rlc_init_microcode(adev, version_major, version_minor);
 711                 if (err)
 712                         goto out;
 713         }
 714
 715         err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw,
 716                                    "amdgpu/%s_mec.bin", ucode_prefix);
 717         if (err)
 718                 goto out;
 719         if (adev->gfx.rs64_enable) {
 720                 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC);
 721                 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P0_STACK);
 722                 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P1_STACK);
 723                 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P2_STACK);
 724                 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P3_STACK);
 725         } else {
 726                 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC1);
 727                 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC1_JT);
 728         }
 729
 730         if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO)
 731                 err = gfx_v11_0_init_toc_microcode(adev, ucode_prefix);
 732
 733         /* only one MEC for gfx 11.0.0. */
 734         adev->gfx.mec2_fw = NULL;
 735
 736         gfx_v11_0_check_fw_cp_gfx_shadow(adev);
 737
 738         if (adev->gfx.imu.funcs && adev->gfx.imu.funcs->init_microcode) {
 739                 err = adev->gfx.imu.funcs->init_microcode(adev);
 740                 if (err)
 741                         DRM_ERROR("Failed to init imu firmware!\n");
 742                 return err;
 743         }
 744
 745 out:
 746         if (err) {
 747                 amdgpu_ucode_release(&adev->gfx.pfp_fw);
 748                 amdgpu_ucode_release(&adev->gfx.me_fw);
 749                 amdgpu_ucode_release(&adev->gfx.rlc_fw);
 750                 amdgpu_ucode_release(&adev->gfx.mec_fw);
 751         }
 752
 753         return err;
 754 }
 755
 756 static u32 gfx_v11_0_get_csb_size(struct amdgpu_device *adev)
 757 {
 758         u32 count = 0;
 759         const struct cs_section_def *sect = NULL;
 760         const struct cs_extent_def *ext = NULL;
 761
 762         /* begin clear state */
 763         count += 2;
 764         /* context control state */
 765         count += 3;
 766
 767         for (sect = gfx11_cs_data; sect->section != NULL; ++sect) {
 768                 for (ext = sect->section; ext->extent != NULL; ++ext) {
 769                         if (sect->id == SECT_CONTEXT)
 770                                 count += 2 + ext->reg_count;
 771                         else
 772                                 return 0;
 773                 }
 774         }
 775
 776         /* set PA_SC_TILE_STEERING_OVERRIDE */
 777         count += 3;
 778         /* end clear state */
 779         count += 2;
 780         /* clear state */
 781         count += 2;
 782
 783         return count;
 784 }
 785
 786 static void gfx_v11_0_get_csb_buffer(struct amdgpu_device *adev,
 787                                     volatile u32 *buffer)
 788 {
 789         u32 count = 0, i;
 790         const struct cs_section_def *sect = NULL;
 791         const struct cs_extent_def *ext = NULL;
 792         int ctx_reg_offset;
 793
 794         if (adev->gfx.rlc.cs_data == NULL)
 795                 return;
 796         if (buffer == NULL)
 797                 return;
 798
 799         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
 800         buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
 801
 802         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
 803         buffer[count++] = cpu_to_le32(0x80000000);
 804         buffer[count++] = cpu_to_le32(0x80000000);
 805
 806         for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) {
 807                 for (ext = sect->section; ext->extent != NULL; ++ext) {
 808                         if (sect->id == SECT_CONTEXT) {
 809                                 buffer[count++] =
 810                                         cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
 811                                 buffer[count++] = cpu_to_le32(ext->reg_index -
 812                                                 PACKET3_SET_CONTEXT_REG_START);
 813                                 for (i = 0; i < ext->reg_count; i++)
 814                                         buffer[count++] = cpu_to_le32(ext->extent[i]);
 815                         } else {
 816                                 return;
 817                         }
 818                 }
 819         }
 820
 821         ctx_reg_offset =
 822                 SOC15_REG_OFFSET(GC, 0, regPA_SC_TILE_STEERING_OVERRIDE) - PACKET3_SET_CONTEXT_REG_START;
 823         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 1));
 824         buffer[count++] = cpu_to_le32(ctx_reg_offset);
 825         buffer[count++] = cpu_to_le32(adev->gfx.config.pa_sc_tile_steering_override);
 826
 827         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
 828         buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
 829
 830         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
 831         buffer[count++] = cpu_to_le32(0);
 832 }
 833
 834 static void gfx_v11_0_rlc_fini(struct amdgpu_device *adev)
 835 {
 836         /* clear state block */
 837         amdgpu_bo_free_kernel(&adev->gfx.rlc.clear_state_obj,
 838                         &adev->gfx.rlc.clear_state_gpu_addr,
 839                         (void **)&adev->gfx.rlc.cs_ptr);
 840
 841         /* jump table block */
 842         amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj,
 843                         &adev->gfx.rlc.cp_table_gpu_addr,
 844                         (void **)&adev->gfx.rlc.cp_table_ptr);
 845 }
 846
 847 static void gfx_v11_0_init_rlcg_reg_access_ctrl(struct amdgpu_device *adev)
 848 {
 849         struct amdgpu_rlcg_reg_access_ctrl *reg_access_ctrl;
 850
 851         reg_access_ctrl = &adev->gfx.rlc.reg_access_ctrl[0];
 852         reg_access_ctrl->scratch_reg0 = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG0);
 853         reg_access_ctrl->scratch_reg1 = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG1);
 854         reg_access_ctrl->scratch_reg2 = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG2);
 855         reg_access_ctrl->scratch_reg3 = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG3);
 856         reg_access_ctrl->grbm_cntl = SOC15_REG_OFFSET(GC, 0, regGRBM_GFX_CNTL);
 857         reg_access_ctrl->grbm_idx = SOC15_REG_OFFSET(GC, 0, regGRBM_GFX_INDEX);
 858         reg_access_ctrl->spare_int = SOC15_REG_OFFSET(GC, 0, regRLC_SPARE_INT_0);
 859         adev->gfx.rlc.rlcg_reg_access_supported = true;
 860 }
 861
 862 static int gfx_v11_0_rlc_init(struct amdgpu_device *adev)
 863 {
 864         const struct cs_section_def *cs_data;
 865         int r;
 866
 867         adev->gfx.rlc.cs_data = gfx11_cs_data;
 868
 869         cs_data = adev->gfx.rlc.cs_data;
 870
 871         if (cs_data) {
 872                 /* init clear state block */
 873                 r = amdgpu_gfx_rlc_init_csb(adev);
 874                 if (r)
 875                         return r;
 876         }
 877
 878         /* init spm vmid with 0xf */
 879         if (adev->gfx.rlc.funcs->update_spm_vmid)
 880                 adev->gfx.rlc.funcs->update_spm_vmid(adev, NULL, 0xf);
 881
 882         return 0;
 883 }
 884
 885 static void gfx_v11_0_mec_fini(struct amdgpu_device *adev)
 886 {
 887         amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL);
 888         amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_obj, NULL, NULL);
 889         amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_data_obj, NULL, NULL);
 890 }
 891
 892 static void gfx_v11_0_me_init(struct amdgpu_device *adev)
 893 {
 894         bitmap_zero(adev->gfx.me.queue_bitmap, AMDGPU_MAX_GFX_QUEUES);
 895
 896         amdgpu_gfx_graphics_queue_acquire(adev);
 897 }
 898
 899 static int gfx_v11_0_mec_init(struct amdgpu_device *adev)
 900 {
 901         int r;
 902         u32 *hpd;
 903         size_t mec_hpd_size;
 904
 905         bitmap_zero(adev->gfx.mec_bitmap[0].queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
 906
 907         /* take ownership of the relevant compute queues */
 908         amdgpu_gfx_compute_queue_acquire(adev);
 909         mec_hpd_size = adev->gfx.num_compute_rings * GFX11_MEC_HPD_SIZE;
 910
 911         if (mec_hpd_size) {
 912                 r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE,
 913                                               AMDGPU_GEM_DOMAIN_GTT,
 914                                               &adev->gfx.mec.hpd_eop_obj,
 915                                               &adev->gfx.mec.hpd_eop_gpu_addr,
 916                                               (void **)&hpd);
 917                 if (r) {
 918                         dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r);
 919                         gfx_v11_0_mec_fini(adev);
 920                         return r;
 921                 }
 922
 923                 memset(hpd, 0, mec_hpd_size);
 924
 925                 amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj);
 926                 amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj);
 927         }
 928
 929         return 0;
 930 }
 931
 932 static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t wave, uint32_t address)
 933 {
 934         WREG32_SOC15(GC, 0, regSQ_IND_INDEX,
 935                 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
 936                 (address << SQ_IND_INDEX__INDEX__SHIFT));
 937         return RREG32_SOC15(GC, 0, regSQ_IND_DATA);
 938 }
 939
 940 static void wave_read_regs(struct amdgpu_device *adev, uint32_t wave,
 941                            uint32_t thread, uint32_t regno,
 942                            uint32_t num, uint32_t *out)
 943 {
 944         WREG32_SOC15(GC, 0, regSQ_IND_INDEX,
 945                 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
 946                 (regno << SQ_IND_INDEX__INDEX__SHIFT) |
 947                 (thread << SQ_IND_INDEX__WORKITEM_ID__SHIFT) |
 948                 (SQ_IND_INDEX__AUTO_INCR_MASK));
 949         while (num--)
 950                 *(out++) = RREG32_SOC15(GC, 0, regSQ_IND_DATA);
 951 }
 952
 953 static void gfx_v11_0_read_wave_data(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields)
 954 {
 955         /* in gfx11 the SIMD_ID is specified as part of the INSTANCE
 956          * field when performing a select_se_sh so it should be
 957          * zero here */
 958         WARN_ON(simd != 0);
 959
 960         /* type 3 wave data */
 961         dst[(*no_fields)++] = 3;
 962         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_STATUS);
 963         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_PC_LO);
 964         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_PC_HI);
 965         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_EXEC_LO);
 966         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_EXEC_HI);
 967         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_HW_ID1);
 968         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_HW_ID2);
 969         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_GPR_ALLOC);
 970         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_LDS_ALLOC);
 971         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_TRAPSTS);
 972         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_IB_STS);
 973         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_IB_STS2);
 974         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_IB_DBG1);
 975         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_M0);
 976         dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_MODE);
 977 }
 978
 979 static void gfx_v11_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd,
 980                                      uint32_t wave, uint32_t start,
 981                                      uint32_t size, uint32_t *dst)
 982 {
 983         WARN_ON(simd != 0);
 984
 985         wave_read_regs(
 986                 adev, wave, 0, start + SQIND_WAVE_SGPRS_OFFSET, size,
 987                 dst);
 988 }
 989
 990 static void gfx_v11_0_read_wave_vgprs(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd,
 991                                       uint32_t wave, uint32_t thread,
 992                                       uint32_t start, uint32_t size,
 993                                       uint32_t *dst)
 994 {
 995         wave_read_regs(
 996                 adev, wave, thread,
 997                 start + SQIND_WAVE_VGPRS_OFFSET, size, dst);
 998 }
 999
1000 static void gfx_v11_0_select_me_pipe_q(struct amdgpu_device *adev,
1001                                         u32 me, u32 pipe, u32 q, u32 vm, u32 xcc_id)
1002 {
1003         soc21_grbm_select(adev, me, pipe, q, vm);
1004 }
1005
1006 /* all sizes are in bytes */
1007 #define MQD_SHADOW_BASE_SIZE      73728
1008 #define MQD_SHADOW_BASE_ALIGNMENT 256
1009 #define MQD_FWWORKAREA_SIZE       484
1010 #define MQD_FWWORKAREA_ALIGNMENT  256
1011
1012 static int gfx_v11_0_get_gfx_shadow_info(struct amdgpu_device *adev,
1013                                          struct amdgpu_gfx_shadow_info *shadow_info)
1014 {
1015         if (adev->gfx.cp_gfx_shadow) {
1016                 shadow_info->shadow_size = MQD_SHADOW_BASE_SIZE;
1017                 shadow_info->shadow_alignment = MQD_SHADOW_BASE_ALIGNMENT;
1018                 shadow_info->csa_size = MQD_FWWORKAREA_SIZE;
1019                 shadow_info->csa_alignment = MQD_FWWORKAREA_ALIGNMENT;
1020                 return 0;
1021         } else {
1022                 memset(shadow_info, 0, sizeof(struct amdgpu_gfx_shadow_info));
1023                 return -ENOTSUPP;
1024         }
1025 }
1026
1027 static const struct amdgpu_gfx_funcs gfx_v11_0_gfx_funcs = {
1028         .get_gpu_clock_counter = &gfx_v11_0_get_gpu_clock_counter,
1029         .select_se_sh = &gfx_v11_0_select_se_sh,
1030         .read_wave_data = &gfx_v11_0_read_wave_data,
1031         .read_wave_sgprs = &gfx_v11_0_read_wave_sgprs,
1032         .read_wave_vgprs = &gfx_v11_0_read_wave_vgprs,
1033         .select_me_pipe_q = &gfx_v11_0_select_me_pipe_q,
1034         .update_perfmon_mgcg = &gfx_v11_0_update_perf_clk,
1035         .get_gfx_shadow_info = &gfx_v11_0_get_gfx_shadow_info,
1036 };
1037
1038 static int gfx_v11_0_gpu_early_init(struct amdgpu_device *adev)
1039 {
1040         switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
1041         case IP_VERSION(11, 0, 0):
1042         case IP_VERSION(11, 0, 2):
1043                 adev->gfx.config.max_hw_contexts = 8;
1044                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1045                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1046                 adev->gfx.config.sc_hiz_tile_fifo_size = 0;
1047                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
1048                 break;
1049         case IP_VERSION(11, 0, 3):
1050                 adev->gfx.ras = &gfx_v11_0_3_ras;
1051                 adev->gfx.config.max_hw_contexts = 8;
1052                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1053                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1054                 adev->gfx.config.sc_hiz_tile_fifo_size = 0;
1055                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
1056                 break;
1057         case IP_VERSION(11, 0, 1):
1058         case IP_VERSION(11, 0, 4):
1059         case IP_VERSION(11, 5, 0):
1060         case IP_VERSION(11, 5, 1):
1061         case IP_VERSION(11, 5, 2):
1062                 adev->gfx.config.max_hw_contexts = 8;
1063                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1064                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1065                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x80;
1066                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x300;
1067                 break;
1068         default:
1069                 BUG();
1070                 break;
1071         }
1072
1073         return 0;
1074 }
1075
1076 static int gfx_v11_0_gfx_ring_init(struct amdgpu_device *adev, int ring_id,
1077                                    int me, int pipe, int queue)
1078 {
1079         struct amdgpu_ring *ring;
1080         unsigned int irq_type;
1081         unsigned int hw_prio;
1082
1083         ring = &adev->gfx.gfx_ring[ring_id];
1084
1085         ring->me = me;
1086         ring->pipe = pipe;
1087         ring->queue = queue;
1088
1089         ring->ring_obj = NULL;
1090         ring->use_doorbell = true;
1091
1092         if (!ring_id)
1093                 ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1;
1094         else
1095                 ring->doorbell_index = adev->doorbell_index.gfx_ring1 << 1;
1096         ring->vm_hub = AMDGPU_GFXHUB(0);
1097         sprintf(ring->name, "gfx_%d.%d.%d", ring->me, ring->pipe, ring->queue);
1098
1099         irq_type = AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP + ring->pipe;
1100         hw_prio = amdgpu_gfx_is_high_priority_graphics_queue(adev, ring) ?
1101                 AMDGPU_GFX_PIPE_PRIO_HIGH : AMDGPU_GFX_PIPE_PRIO_NORMAL;
1102         return amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type,
1103                                 hw_prio, NULL);
1104 }
1105
1106 static int gfx_v11_0_compute_ring_init(struct amdgpu_device *adev, int ring_id,
1107                                        int mec, int pipe, int queue)
1108 {
1109         int r;
1110         unsigned irq_type;
1111         struct amdgpu_ring *ring;
1112         unsigned int hw_prio;
1113
1114         ring = &adev->gfx.compute_ring[ring_id];
1115
1116         /* mec0 is me1 */
1117         ring->me = mec + 1;
1118         ring->pipe = pipe;
1119         ring->queue = queue;
1120
1121         ring->ring_obj = NULL;
1122         ring->use_doorbell = true;
1123         ring->doorbell_index = (adev->doorbell_index.mec_ring0 + ring_id) << 1;
1124         ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr
1125                                 + (ring_id * GFX11_MEC_HPD_SIZE);
1126         ring->vm_hub = AMDGPU_GFXHUB(0);
1127         sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue);
1128
1129         irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
1130                 + ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec)
1131                 + ring->pipe;
1132         hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring) ?
1133                         AMDGPU_GFX_PIPE_PRIO_HIGH : AMDGPU_GFX_PIPE_PRIO_NORMAL;
1134         /* type-2 packets are deprecated on MEC, use type-3 instead */
1135         r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type,
1136                              hw_prio, NULL);
1137         if (r)
1138                 return r;
1139
1140         return 0;
1141 }
1142
1143 static struct {
1144         SOC21_FIRMWARE_ID       id;
1145         unsigned int            offset;
1146         unsigned int            size;
1147 } rlc_autoload_info[SOC21_FIRMWARE_ID_MAX];
1148
1149 static void gfx_v11_0_parse_rlc_toc(struct amdgpu_device *adev, void *rlc_toc)
1150 {
1151         RLC_TABLE_OF_CONTENT *ucode = rlc_toc;
1152
1153         while (ucode && (ucode->id > SOC21_FIRMWARE_ID_INVALID) &&
1154                         (ucode->id < SOC21_FIRMWARE_ID_MAX)) {
1155                 rlc_autoload_info[ucode->id].id = ucode->id;
1156                 rlc_autoload_info[ucode->id].offset = ucode->offset * 4;
1157                 rlc_autoload_info[ucode->id].size = ucode->size * 4;
1158
1159                 ucode++;
1160         }
1161 }
1162
1163 static uint32_t gfx_v11_0_calc_toc_total_size(struct amdgpu_device *adev)
1164 {
1165         uint32_t total_size = 0;
1166         SOC21_FIRMWARE_ID id;
1167
1168         gfx_v11_0_parse_rlc_toc(adev, adev->psp.toc.start_addr);
1169
1170         for (id = SOC21_FIRMWARE_ID_RLC_G_UCODE; id < SOC21_FIRMWARE_ID_MAX; id++)
1171                 total_size += rlc_autoload_info[id].size;
1172
1173         /* In case the offset in rlc toc ucode is aligned */
1174         if (total_size < rlc_autoload_info[SOC21_FIRMWARE_ID_MAX-1].offset)
1175                 total_size = rlc_autoload_info[SOC21_FIRMWARE_ID_MAX-1].offset +
1176                         rlc_autoload_info[SOC21_FIRMWARE_ID_MAX-1].size;
1177
1178         return total_size;
1179 }
1180
1181 static int gfx_v11_0_rlc_autoload_buffer_init(struct amdgpu_device *adev)
1182 {
1183         int r;
1184         uint32_t total_size;
1185
1186         total_size = gfx_v11_0_calc_toc_total_size(adev);
1187
1188         r = amdgpu_bo_create_reserved(adev, total_size, 64 * 1024,
1189                                       AMDGPU_GEM_DOMAIN_VRAM |
1190                                       AMDGPU_GEM_DOMAIN_GTT,
1191                                       &adev->gfx.rlc.rlc_autoload_bo,
1192                                       &adev->gfx.rlc.rlc_autoload_gpu_addr,
1193                                       (void **)&adev->gfx.rlc.rlc_autoload_ptr);
1194
1195         if (r) {
1196                 dev_err(adev->dev, "(%d) failed to create fw autoload bo\n", r);
1197                 return r;
1198         }
1199
1200         return 0;
1201 }
1202
1203 static void gfx_v11_0_rlc_backdoor_autoload_copy_ucode(struct amdgpu_device *adev,
1204                                               SOC21_FIRMWARE_ID id,
1205                                               const void *fw_data,
1206                                               uint32_t fw_size,
1207                                               uint32_t *fw_autoload_mask)
1208 {
1209         uint32_t toc_offset;
1210         uint32_t toc_fw_size;
1211         char *ptr = adev->gfx.rlc.rlc_autoload_ptr;
1212
1213         if (id <= SOC21_FIRMWARE_ID_INVALID || id >= SOC21_FIRMWARE_ID_MAX)
1214                 return;
1215
1216         toc_offset = rlc_autoload_info[id].offset;
1217         toc_fw_size = rlc_autoload_info[id].size;
1218
1219         if (fw_size == 0)
1220                 fw_size = toc_fw_size;
1221
1222         if (fw_size > toc_fw_size)
1223                 fw_size = toc_fw_size;
1224
1225         memcpy(ptr + toc_offset, fw_data, fw_size);
1226
1227         if (fw_size < toc_fw_size)
1228                 memset(ptr + toc_offset + fw_size, 0, toc_fw_size - fw_size);
1229
1230         if ((id != SOC21_FIRMWARE_ID_RS64_PFP) && (id != SOC21_FIRMWARE_ID_RS64_ME))
1231                 *(uint64_t *)fw_autoload_mask |= 1ULL << id;
1232 }
1233
1234 static void gfx_v11_0_rlc_backdoor_autoload_copy_toc_ucode(struct amdgpu_device *adev,
1235                                                         uint32_t *fw_autoload_mask)
1236 {
1237         void *data;
1238         uint32_t size;
1239         uint64_t *toc_ptr;
1240
1241         *(uint64_t *)fw_autoload_mask |= 0x1;
1242
1243         DRM_DEBUG("rlc autoload enabled fw: 0x%llx\n", *(uint64_t *)fw_autoload_mask);
1244
1245         data = adev->psp.toc.start_addr;
1246         size = rlc_autoload_info[SOC21_FIRMWARE_ID_RLC_TOC].size;
1247
1248         toc_ptr = (uint64_t *)data + size / 8 - 1;
1249         *toc_ptr = *(uint64_t *)fw_autoload_mask;
1250
1251         gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RLC_TOC,
1252                                         data, size, fw_autoload_mask);
1253 }
1254
1255 static void gfx_v11_0_rlc_backdoor_autoload_copy_gfx_ucode(struct amdgpu_device *adev,
1256                                                         uint32_t *fw_autoload_mask)
1257 {
1258         const __le32 *fw_data;
1259         uint32_t fw_size;
1260         const struct gfx_firmware_header_v1_0 *cp_hdr;
1261         const struct gfx_firmware_header_v2_0 *cpv2_hdr;
1262         const struct rlc_firmware_header_v2_0 *rlc_hdr;
1263         const struct rlc_firmware_header_v2_2 *rlcv22_hdr;
1264         uint16_t version_major, version_minor;
1265
1266         if (adev->gfx.rs64_enable) {
1267                 /* pfp ucode */
1268                 cpv2_hdr = (const struct gfx_firmware_header_v2_0 *)
1269                         adev->gfx.pfp_fw->data;
1270                 /* instruction */
1271                 fw_data = (const __le32 *)(adev->gfx.pfp_fw->data +
1272                         le32_to_cpu(cpv2_hdr->ucode_offset_bytes));
1273                 fw_size = le32_to_cpu(cpv2_hdr->ucode_size_bytes);
1274                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_PFP,
1275                                                 fw_data, fw_size, fw_autoload_mask);
1276                 /* data */
1277                 fw_data = (const __le32 *)(adev->gfx.pfp_fw->data +
1278                         le32_to_cpu(cpv2_hdr->data_offset_bytes));
1279                 fw_size = le32_to_cpu(cpv2_hdr->data_size_bytes);
1280                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_PFP_P0_STACK,
1281                                                 fw_data, fw_size, fw_autoload_mask);
1282                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_PFP_P1_STACK,
1283                                                 fw_data, fw_size, fw_autoload_mask);
1284                 /* me ucode */
1285                 cpv2_hdr = (const struct gfx_firmware_header_v2_0 *)
1286                         adev->gfx.me_fw->data;
1287                 /* instruction */
1288                 fw_data = (const __le32 *)(adev->gfx.me_fw->data +
1289                         le32_to_cpu(cpv2_hdr->ucode_offset_bytes));
1290                 fw_size = le32_to_cpu(cpv2_hdr->ucode_size_bytes);
1291                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_ME,
1292                                                 fw_data, fw_size, fw_autoload_mask);
1293                 /* data */
1294                 fw_data = (const __le32 *)(adev->gfx.me_fw->data +
1295                         le32_to_cpu(cpv2_hdr->data_offset_bytes));
1296                 fw_size = le32_to_cpu(cpv2_hdr->data_size_bytes);
1297                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_ME_P0_STACK,
1298                                                 fw_data, fw_size, fw_autoload_mask);
1299                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_ME_P1_STACK,
1300                                                 fw_data, fw_size, fw_autoload_mask);
1301                 /* mec ucode */
1302                 cpv2_hdr = (const struct gfx_firmware_header_v2_0 *)
1303                         adev->gfx.mec_fw->data;
1304                 /* instruction */
1305                 fw_data = (const __le32 *) (adev->gfx.mec_fw->data +
1306                         le32_to_cpu(cpv2_hdr->ucode_offset_bytes));
1307                 fw_size = le32_to_cpu(cpv2_hdr->ucode_size_bytes);
1308                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_MEC,
1309                                                 fw_data, fw_size, fw_autoload_mask);
1310                 /* data */
1311                 fw_data = (const __le32 *) (adev->gfx.mec_fw->data +
1312                         le32_to_cpu(cpv2_hdr->data_offset_bytes));
1313                 fw_size = le32_to_cpu(cpv2_hdr->data_size_bytes);
1314                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_MEC_P0_STACK,
1315                                                 fw_data, fw_size, fw_autoload_mask);
1316                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_MEC_P1_STACK,
1317                                                 fw_data, fw_size, fw_autoload_mask);
1318                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_MEC_P2_STACK,
1319                                                 fw_data, fw_size, fw_autoload_mask);
1320                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_MEC_P3_STACK,
1321                                                 fw_data, fw_size, fw_autoload_mask);
1322         } else {
1323                 /* pfp ucode */
1324                 cp_hdr = (const struct gfx_firmware_header_v1_0 *)
1325                         adev->gfx.pfp_fw->data;
1326                 fw_data = (const __le32 *)(adev->gfx.pfp_fw->data +
1327                                 le32_to_cpu(cp_hdr->header.ucode_array_offset_bytes));
1328                 fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes);
1329                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_CP_PFP,
1330                                                 fw_data, fw_size, fw_autoload_mask);
1331
1332                 /* me ucode */
1333                 cp_hdr = (const struct gfx_firmware_header_v1_0 *)
1334                         adev->gfx.me_fw->data;
1335                 fw_data = (const __le32 *)(adev->gfx.me_fw->data +
1336                                 le32_to_cpu(cp_hdr->header.ucode_array_offset_bytes));
1337                 fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes);
1338                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_CP_ME,
1339                                                 fw_data, fw_size, fw_autoload_mask);
1340
1341                 /* mec ucode */
1342                 cp_hdr = (const struct gfx_firmware_header_v1_0 *)
1343                         adev->gfx.mec_fw->data;
1344                 fw_data = (const __le32 *) (adev->gfx.mec_fw->data +
1345                                 le32_to_cpu(cp_hdr->header.ucode_array_offset_bytes));
1346                 fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes) -
1347                         cp_hdr->jt_size * 4;
1348                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_CP_MEC,
1349                                                 fw_data, fw_size, fw_autoload_mask);
1350         }
1351
1352         /* rlc ucode */
1353         rlc_hdr = (const struct rlc_firmware_header_v2_0 *)
1354                 adev->gfx.rlc_fw->data;
1355         fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1356                         le32_to_cpu(rlc_hdr->header.ucode_array_offset_bytes));
1357         fw_size = le32_to_cpu(rlc_hdr->header.ucode_size_bytes);
1358         gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RLC_G_UCODE,
1359                                         fw_data, fw_size, fw_autoload_mask);
1360
1361         version_major = le16_to_cpu(rlc_hdr->header.header_version_major);
1362         version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor);
1363         if (version_major == 2) {
1364                 if (version_minor >= 2) {
1365                         rlcv22_hdr = (const struct rlc_firmware_header_v2_2 *)adev->gfx.rlc_fw->data;
1366
1367                         fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1368                                         le32_to_cpu(rlcv22_hdr->rlc_iram_ucode_offset_bytes));
1369                         fw_size = le32_to_cpu(rlcv22_hdr->rlc_iram_ucode_size_bytes);
1370                         gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RLX6_UCODE,
1371                                         fw_data, fw_size, fw_autoload_mask);
1372
1373                         fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1374                                         le32_to_cpu(rlcv22_hdr->rlc_dram_ucode_offset_bytes));
1375                         fw_size = le32_to_cpu(rlcv22_hdr->rlc_dram_ucode_size_bytes);
1376                         gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RLX6_DRAM_BOOT,
1377                                         fw_data, fw_size, fw_autoload_mask);
1378                 }
1379         }
1380 }
1381
1382 static void gfx_v11_0_rlc_backdoor_autoload_copy_sdma_ucode(struct amdgpu_device *adev,
1383                                                         uint32_t *fw_autoload_mask)
1384 {
1385         const __le32 *fw_data;
1386         uint32_t fw_size;
1387         const struct sdma_firmware_header_v2_0 *sdma_hdr;
1388
1389         sdma_hdr = (const struct sdma_firmware_header_v2_0 *)
1390                 adev->sdma.instance[0].fw->data;
1391         fw_data = (const __le32 *) (adev->sdma.instance[0].fw->data +
1392                         le32_to_cpu(sdma_hdr->header.ucode_array_offset_bytes));
1393         fw_size = le32_to_cpu(sdma_hdr->ctx_ucode_size_bytes);
1394
1395         gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev,
1396                         SOC21_FIRMWARE_ID_SDMA_UCODE_TH0, fw_data, fw_size, fw_autoload_mask);
1397
1398         fw_data = (const __le32 *) (adev->sdma.instance[0].fw->data +
1399                         le32_to_cpu(sdma_hdr->ctl_ucode_offset));
1400         fw_size = le32_to_cpu(sdma_hdr->ctl_ucode_size_bytes);
1401
1402         gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev,
1403                         SOC21_FIRMWARE_ID_SDMA_UCODE_TH1, fw_data, fw_size, fw_autoload_mask);
1404 }
1405
1406 static void gfx_v11_0_rlc_backdoor_autoload_copy_mes_ucode(struct amdgpu_device *adev,
1407                                                         uint32_t *fw_autoload_mask)
1408 {
1409         const __le32 *fw_data;
1410         unsigned fw_size;
1411         const struct mes_firmware_header_v1_0 *mes_hdr;
1412         int pipe, ucode_id, data_id;
1413
1414         for (pipe = 0; pipe < 2; pipe++) {
1415                 if (pipe==0) {
1416                         ucode_id = SOC21_FIRMWARE_ID_RS64_MES_P0;
1417                         data_id  = SOC21_FIRMWARE_ID_RS64_MES_P0_STACK;
1418                 } else {
1419                         ucode_id = SOC21_FIRMWARE_ID_RS64_MES_P1;
1420                         data_id  = SOC21_FIRMWARE_ID_RS64_MES_P1_STACK;
1421                 }
1422
1423                 mes_hdr = (const struct mes_firmware_header_v1_0 *)
1424                         adev->mes.fw[pipe]->data;
1425
1426                 fw_data = (const __le32 *)(adev->mes.fw[pipe]->data +
1427                                 le32_to_cpu(mes_hdr->mes_ucode_offset_bytes));
1428                 fw_size = le32_to_cpu(mes_hdr->mes_ucode_size_bytes);
1429
1430                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev,
1431                                 ucode_id, fw_data, fw_size, fw_autoload_mask);
1432
1433                 fw_data = (const __le32 *)(adev->mes.fw[pipe]->data +
1434                                 le32_to_cpu(mes_hdr->mes_ucode_data_offset_bytes));
1435                 fw_size = le32_to_cpu(mes_hdr->mes_ucode_data_size_bytes);
1436
1437                 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev,
1438                                 data_id, fw_data, fw_size, fw_autoload_mask);
1439         }
1440 }
1441
1442 static int gfx_v11_0_rlc_backdoor_autoload_enable(struct amdgpu_device *adev)
1443 {
1444         uint32_t rlc_g_offset, rlc_g_size;
1445         uint64_t gpu_addr;
1446         uint32_t autoload_fw_id[2];
1447
1448         memset(autoload_fw_id, 0, sizeof(uint32_t) * 2);
1449
1450         /* RLC autoload sequence 2: copy ucode */
1451         gfx_v11_0_rlc_backdoor_autoload_copy_sdma_ucode(adev, autoload_fw_id);
1452         gfx_v11_0_rlc_backdoor_autoload_copy_gfx_ucode(adev, autoload_fw_id);
1453         gfx_v11_0_rlc_backdoor_autoload_copy_mes_ucode(adev, autoload_fw_id);
1454         gfx_v11_0_rlc_backdoor_autoload_copy_toc_ucode(adev, autoload_fw_id);
1455
1456         rlc_g_offset = rlc_autoload_info[SOC21_FIRMWARE_ID_RLC_G_UCODE].offset;
1457         rlc_g_size = rlc_autoload_info[SOC21_FIRMWARE_ID_RLC_G_UCODE].size;
1458         gpu_addr = adev->gfx.rlc.rlc_autoload_gpu_addr + rlc_g_offset;
1459
1460         WREG32_SOC15(GC, 0, regGFX_IMU_RLC_BOOTLOADER_ADDR_HI, upper_32_bits(gpu_addr));
1461         WREG32_SOC15(GC, 0, regGFX_IMU_RLC_BOOTLOADER_ADDR_LO, lower_32_bits(gpu_addr));
1462
1463         WREG32_SOC15(GC, 0, regGFX_IMU_RLC_BOOTLOADER_SIZE, rlc_g_size);
1464
1465         /* RLC autoload sequence 3: load IMU fw */
1466         if (adev->gfx.imu.funcs->load_microcode)
1467                 adev->gfx.imu.funcs->load_microcode(adev);
1468         /* RLC autoload sequence 4 init IMU fw */
1469         if (adev->gfx.imu.funcs->setup_imu)
1470                 adev->gfx.imu.funcs->setup_imu(adev);
1471         if (adev->gfx.imu.funcs->start_imu)
1472                 adev->gfx.imu.funcs->start_imu(adev);
1473
1474         /* RLC autoload sequence 5 disable gpa mode */
1475         gfx_v11_0_disable_gpa_mode(adev);
1476
1477         return 0;
1478 }
1479
1480 static void gfx_v11_0_alloc_ip_dump(struct amdgpu_device *adev)
1481 {
1482         uint32_t reg_count = ARRAY_SIZE(gc_reg_list_11_0);
1483         uint32_t *ptr;
1484         uint32_t inst;
1485
1486         ptr = kcalloc(reg_count, sizeof(uint32_t), GFP_KERNEL);
1487         if (ptr == NULL) {
1488                 DRM_ERROR("Failed to allocate memory for GFX IP Dump\n");
1489                 adev->gfx.ip_dump_core = NULL;
1490         } else {
1491                 adev->gfx.ip_dump_core = ptr;
1492         }
1493
1494         /* Allocate memory for compute queue registers for all the instances */
1495         reg_count = ARRAY_SIZE(gc_cp_reg_list_11);
1496         inst = adev->gfx.mec.num_mec * adev->gfx.mec.num_pipe_per_mec *
1497                 adev->gfx.mec.num_queue_per_pipe;
1498
1499         ptr = kcalloc(reg_count * inst, sizeof(uint32_t), GFP_KERNEL);
1500         if (ptr == NULL) {
1501                 DRM_ERROR("Failed to allocate memory for Compute Queues IP Dump\n");
1502                 adev->gfx.ip_dump_compute_queues = NULL;
1503         } else {
1504                 adev->gfx.ip_dump_compute_queues = ptr;
1505         }
1506
1507         /* Allocate memory for gfx queue registers for all the instances */
1508         reg_count = ARRAY_SIZE(gc_gfx_queue_reg_list_11);
1509         inst = adev->gfx.me.num_me * adev->gfx.me.num_pipe_per_me *
1510                 adev->gfx.me.num_queue_per_pipe;
1511
1512         ptr = kcalloc(reg_count * inst, sizeof(uint32_t), GFP_KERNEL);
1513         if (ptr == NULL) {
1514                 DRM_ERROR("Failed to allocate memory for GFX Queues IP Dump\n");
1515                 adev->gfx.ip_dump_gfx_queues = NULL;
1516         } else {
1517                 adev->gfx.ip_dump_gfx_queues = ptr;
1518         }
1519 }
1520
1521 static int gfx_v11_0_sw_init(void *handle)
1522 {
1523         int i, j, k, r, ring_id = 0;
1524         int xcc_id = 0;
1525         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1526
1527         switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
1528         case IP_VERSION(11, 0, 0):
1529         case IP_VERSION(11, 0, 2):
1530         case IP_VERSION(11, 0, 3):
1531                 adev->gfx.me.num_me = 1;
1532                 adev->gfx.me.num_pipe_per_me = 1;
1533                 adev->gfx.me.num_queue_per_pipe = 1;
1534                 adev->gfx.mec.num_mec = 2;
1535                 adev->gfx.mec.num_pipe_per_mec = 4;
1536                 adev->gfx.mec.num_queue_per_pipe = 4;
1537                 break;
1538         case IP_VERSION(11, 0, 1):
1539         case IP_VERSION(11, 0, 4):
1540         case IP_VERSION(11, 5, 0):
1541         case IP_VERSION(11, 5, 1):
1542         case IP_VERSION(11, 5, 2):
1543                 adev->gfx.me.num_me = 1;
1544                 adev->gfx.me.num_pipe_per_me = 1;
1545                 adev->gfx.me.num_queue_per_pipe = 1;
1546                 adev->gfx.mec.num_mec = 1;
1547                 adev->gfx.mec.num_pipe_per_mec = 4;
1548                 adev->gfx.mec.num_queue_per_pipe = 4;
1549                 break;
1550         default:
1551                 adev->gfx.me.num_me = 1;
1552                 adev->gfx.me.num_pipe_per_me = 1;
1553                 adev->gfx.me.num_queue_per_pipe = 1;
1554                 adev->gfx.mec.num_mec = 1;
1555                 adev->gfx.mec.num_pipe_per_mec = 4;
1556                 adev->gfx.mec.num_queue_per_pipe = 8;
1557                 break;
1558         }
1559
1560         /* Enable CG flag in one VF mode for enabling RLC safe mode enter/exit */
1561         if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 0, 3) &&
1562             amdgpu_sriov_is_pp_one_vf(adev))
1563                 adev->cg_flags = AMD_CG_SUPPORT_GFX_CGCG;
1564
1565         /* EOP Event */
1566         r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GRBM_CP,
1567                               GFX_11_0_0__SRCID__CP_EOP_INTERRUPT,
1568                               &adev->gfx.eop_irq);
1569         if (r)
1570                 return r;
1571
1572         /* Privileged reg */
1573         r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GRBM_CP,
1574                               GFX_11_0_0__SRCID__CP_PRIV_REG_FAULT,
1575                               &adev->gfx.priv_reg_irq);
1576         if (r)
1577                 return r;
1578
1579         /* Privileged inst */
1580         r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GRBM_CP,
1581                               GFX_11_0_0__SRCID__CP_PRIV_INSTR_FAULT,
1582                               &adev->gfx.priv_inst_irq);
1583         if (r)
1584                 return r;
1585
1586         /* FED error */
1587         r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GFX,
1588                                   GFX_11_0_0__SRCID__RLC_GC_FED_INTERRUPT,
1589                                   &adev->gfx.rlc_gc_fed_irq);
1590         if (r)
1591                 return r;
1592
1593         adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE;
1594
1595         gfx_v11_0_me_init(adev);
1596
1597         r = gfx_v11_0_rlc_init(adev);
1598         if (r) {
1599                 DRM_ERROR("Failed to init rlc BOs!\n");
1600                 return r;
1601         }
1602
1603         r = gfx_v11_0_mec_init(adev);
1604         if (r) {
1605                 DRM_ERROR("Failed to init MEC BOs!\n");
1606                 return r;
1607         }
1608
1609         /* set up the gfx ring */
1610         for (i = 0; i < adev->gfx.me.num_me; i++) {
1611                 for (j = 0; j < adev->gfx.me.num_queue_per_pipe; j++) {
1612                         for (k = 0; k < adev->gfx.me.num_pipe_per_me; k++) {
1613                                 if (!amdgpu_gfx_is_me_queue_enabled(adev, i, k, j))
1614                                         continue;
1615
1616                                 r = gfx_v11_0_gfx_ring_init(adev, ring_id,
1617                                                             i, k, j);
1618                                 if (r)
1619                                         return r;
1620                                 ring_id++;
1621                         }
1622                 }
1623         }
1624
1625         ring_id = 0;
1626         /* set up the compute queues - allocate horizontally across pipes */
1627         for (i = 0; i < adev->gfx.mec.num_mec; ++i) {
1628                 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {
1629                         for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {
1630                                 if (!amdgpu_gfx_is_mec_queue_enabled(adev, 0, i,
1631                                                                      k, j))
1632                                         continue;
1633
1634                                 r = gfx_v11_0_compute_ring_init(adev, ring_id,
1635                                                                 i, k, j);
1636                                 if (r)
1637                                         return r;
1638
1639                                 ring_id++;
1640                         }
1641                 }
1642         }
1643
1644         if (!adev->enable_mes_kiq) {
1645                 r = amdgpu_gfx_kiq_init(adev, GFX11_MEC_HPD_SIZE, 0);
1646                 if (r) {
1647                         DRM_ERROR("Failed to init KIQ BOs!\n");
1648                         return r;
1649                 }
1650
1651                 r = amdgpu_gfx_kiq_init_ring(adev, xcc_id);
1652                 if (r)
1653                         return r;
1654         }
1655
1656         r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct v11_compute_mqd), 0);
1657         if (r)
1658                 return r;
1659
1660         /* allocate visible FB for rlc auto-loading fw */
1661         if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) {
1662                 r = gfx_v11_0_rlc_autoload_buffer_init(adev);
1663                 if (r)
1664                         return r;
1665         }
1666
1667         r = gfx_v11_0_gpu_early_init(adev);
1668         if (r)
1669                 return r;
1670
1671         if (amdgpu_gfx_ras_sw_init(adev)) {
1672                 dev_err(adev->dev, "Failed to initialize gfx ras block!\n");
1673                 return -EINVAL;
1674         }
1675
1676         gfx_v11_0_alloc_ip_dump(adev);
1677
1678         return 0;
1679 }
1680
1681 static void gfx_v11_0_pfp_fini(struct amdgpu_device *adev)
1682 {
1683         amdgpu_bo_free_kernel(&adev->gfx.pfp.pfp_fw_obj,
1684                               &adev->gfx.pfp.pfp_fw_gpu_addr,
1685                               (void **)&adev->gfx.pfp.pfp_fw_ptr);
1686
1687         amdgpu_bo_free_kernel(&adev->gfx.pfp.pfp_fw_data_obj,
1688                               &adev->gfx.pfp.pfp_fw_data_gpu_addr,
1689                               (void **)&adev->gfx.pfp.pfp_fw_data_ptr);
1690 }
1691
1692 static void gfx_v11_0_me_fini(struct amdgpu_device *adev)
1693 {
1694         amdgpu_bo_free_kernel(&adev->gfx.me.me_fw_obj,
1695                               &adev->gfx.me.me_fw_gpu_addr,
1696                               (void **)&adev->gfx.me.me_fw_ptr);
1697
1698         amdgpu_bo_free_kernel(&adev->gfx.me.me_fw_data_obj,
1699                                &adev->gfx.me.me_fw_data_gpu_addr,
1700                                (void **)&adev->gfx.me.me_fw_data_ptr);
1701 }
1702
1703 static void gfx_v11_0_rlc_autoload_buffer_fini(struct amdgpu_device *adev)
1704 {
1705         amdgpu_bo_free_kernel(&adev->gfx.rlc.rlc_autoload_bo,
1706                         &adev->gfx.rlc.rlc_autoload_gpu_addr,
1707                         (void **)&adev->gfx.rlc.rlc_autoload_ptr);
1708 }
1709
1710 static int gfx_v11_0_sw_fini(void *handle)
1711 {
1712         int i;
1713         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1714
1715         for (i = 0; i < adev->gfx.num_gfx_rings; i++)
1716                 amdgpu_ring_fini(&adev->gfx.gfx_ring[i]);
1717         for (i = 0; i < adev->gfx.num_compute_rings; i++)
1718                 amdgpu_ring_fini(&adev->gfx.compute_ring[i]);
1719
1720         amdgpu_gfx_mqd_sw_fini(adev, 0);
1721
1722         if (!adev->enable_mes_kiq) {
1723                 amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq[0].ring);
1724                 amdgpu_gfx_kiq_fini(adev, 0);
1725         }
1726
1727         gfx_v11_0_pfp_fini(adev);
1728         gfx_v11_0_me_fini(adev);
1729         gfx_v11_0_rlc_fini(adev);
1730         gfx_v11_0_mec_fini(adev);
1731
1732         if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO)
1733                 gfx_v11_0_rlc_autoload_buffer_fini(adev);
1734
1735         gfx_v11_0_free_microcode(adev);
1736
1737         kfree(adev->gfx.ip_dump_core);
1738         kfree(adev->gfx.ip_dump_compute_queues);
1739         kfree(adev->gfx.ip_dump_gfx_queues);
1740
1741         return 0;
1742 }
1743
1744 static void gfx_v11_0_select_se_sh(struct amdgpu_device *adev, u32 se_num,
1745                                    u32 sh_num, u32 instance, int xcc_id)
1746 {
1747         u32 data;
1748
1749         if (instance == 0xffffffff)
1750                 data = REG_SET_FIELD(0, GRBM_GFX_INDEX,
1751                                      INSTANCE_BROADCAST_WRITES, 1);
1752         else
1753                 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX,
1754                                      instance);
1755
1756         if (se_num == 0xffffffff)
1757                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES,
1758                                      1);
1759         else
1760                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num);
1761
1762         if (sh_num == 0xffffffff)
1763                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SA_BROADCAST_WRITES,
1764                                      1);
1765         else
1766                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SA_INDEX, sh_num);
1767
1768         WREG32_SOC15(GC, 0, regGRBM_GFX_INDEX, data);
1769 }
1770
1771 static u32 gfx_v11_0_get_sa_active_bitmap(struct amdgpu_device *adev)
1772 {
1773         u32 gc_disabled_sa_mask, gc_user_disabled_sa_mask, sa_mask;
1774
1775         gc_disabled_sa_mask = RREG32_SOC15(GC, 0, regCC_GC_SA_UNIT_DISABLE);
1776         gc_disabled_sa_mask = REG_GET_FIELD(gc_disabled_sa_mask,
1777                                            CC_GC_SA_UNIT_DISABLE,
1778                                            SA_DISABLE);
1779         gc_user_disabled_sa_mask = RREG32_SOC15(GC, 0, regGC_USER_SA_UNIT_DISABLE);
1780         gc_user_disabled_sa_mask = REG_GET_FIELD(gc_user_disabled_sa_mask,
1781                                                  GC_USER_SA_UNIT_DISABLE,
1782                                                  SA_DISABLE);
1783         sa_mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_sh_per_se *
1784                                             adev->gfx.config.max_shader_engines);
1785
1786         return sa_mask & (~(gc_disabled_sa_mask | gc_user_disabled_sa_mask));
1787 }
1788
1789 static u32 gfx_v11_0_get_rb_active_bitmap(struct amdgpu_device *adev)
1790 {
1791         u32 gc_disabled_rb_mask, gc_user_disabled_rb_mask;
1792         u32 rb_mask;
1793
1794         gc_disabled_rb_mask = RREG32_SOC15(GC, 0, regCC_RB_BACKEND_DISABLE);
1795         gc_disabled_rb_mask = REG_GET_FIELD(gc_disabled_rb_mask,
1796                                             CC_RB_BACKEND_DISABLE,
1797                                             BACKEND_DISABLE);
1798         gc_user_disabled_rb_mask = RREG32_SOC15(GC, 0, regGC_USER_RB_BACKEND_DISABLE);
1799         gc_user_disabled_rb_mask = REG_GET_FIELD(gc_user_disabled_rb_mask,
1800                                                  GC_USER_RB_BACKEND_DISABLE,
1801                                                  BACKEND_DISABLE);
1802         rb_mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se *
1803                                             adev->gfx.config.max_shader_engines);
1804
1805         return rb_mask & (~(gc_disabled_rb_mask | gc_user_disabled_rb_mask));
1806 }
1807
1808 static void gfx_v11_0_setup_rb(struct amdgpu_device *adev)
1809 {
1810         u32 rb_bitmap_width_per_sa;
1811         u32 max_sa;
1812         u32 active_sa_bitmap;
1813         u32 global_active_rb_bitmap;
1814         u32 active_rb_bitmap = 0;
1815         u32 i;
1816
1817         /* query sa bitmap from SA_UNIT_DISABLE registers */
1818         active_sa_bitmap = gfx_v11_0_get_sa_active_bitmap(adev);
1819         /* query rb bitmap from RB_BACKEND_DISABLE registers */
1820         global_active_rb_bitmap = gfx_v11_0_get_rb_active_bitmap(adev);
1821
1822         /* generate active rb bitmap according to active sa bitmap */
1823         max_sa = adev->gfx.config.max_shader_engines *
1824                  adev->gfx.config.max_sh_per_se;
1825         rb_bitmap_width_per_sa = adev->gfx.config.max_backends_per_se /
1826                                  adev->gfx.config.max_sh_per_se;
1827         for (i = 0; i < max_sa; i++) {
1828                 if (active_sa_bitmap & (1 << i))
1829                         active_rb_bitmap |= (0x3 << (i * rb_bitmap_width_per_sa));
1830         }
1831
1832         active_rb_bitmap &= global_active_rb_bitmap;
1833         adev->gfx.config.backend_enable_mask = active_rb_bitmap;
1834         adev->gfx.config.num_rbs = hweight32(active_rb_bitmap);
1835 }
1836
1837 #define DEFAULT_SH_MEM_BASES    (0x6000)
1838 #define LDS_APP_BASE           0x1
1839 #define SCRATCH_APP_BASE       0x2
1840
1841 static void gfx_v11_0_init_compute_vmid(struct amdgpu_device *adev)
1842 {
1843         int i;
1844         uint32_t sh_mem_bases;
1845         uint32_t data;
1846
1847         /*
1848          * Configure apertures:
1849          * LDS:         0x60000000'00000000 - 0x60000001'00000000 (4GB)
1850          * Scratch:     0x60000001'00000000 - 0x60000002'00000000 (4GB)
1851          * GPUVM:       0x60010000'00000000 - 0x60020000'00000000 (1TB)
1852          */
1853         sh_mem_bases = (LDS_APP_BASE << SH_MEM_BASES__SHARED_BASE__SHIFT) |
1854                         SCRATCH_APP_BASE;
1855
1856         mutex_lock(&adev->srbm_mutex);
1857         for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
1858                 soc21_grbm_select(adev, 0, 0, 0, i);
1859                 /* CP and shaders */
1860                 WREG32_SOC15(GC, 0, regSH_MEM_CONFIG, DEFAULT_SH_MEM_CONFIG);
1861                 WREG32_SOC15(GC, 0, regSH_MEM_BASES, sh_mem_bases);
1862
1863                 /* Enable trap for each kfd vmid. */
1864                 data = RREG32_SOC15(GC, 0, regSPI_GDBG_PER_VMID_CNTL);
1865                 data = REG_SET_FIELD(data, SPI_GDBG_PER_VMID_CNTL, TRAP_EN, 1);
1866                 WREG32_SOC15(GC, 0, regSPI_GDBG_PER_VMID_CNTL, data);
1867         }
1868         soc21_grbm_select(adev, 0, 0, 0, 0);
1869         mutex_unlock(&adev->srbm_mutex);
1870
1871         /* Initialize all compute VMIDs to have no GDS, GWS, or OA
1872            acccess. These should be enabled by FW for target VMIDs. */
1873         for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
1874                 WREG32_SOC15_OFFSET(GC, 0, regGDS_VMID0_BASE, 2 * i, 0);
1875                 WREG32_SOC15_OFFSET(GC, 0, regGDS_VMID0_SIZE, 2 * i, 0);
1876                 WREG32_SOC15_OFFSET(GC, 0, regGDS_GWS_VMID0, i, 0);
1877                 WREG32_SOC15_OFFSET(GC, 0, regGDS_OA_VMID0, i, 0);
1878         }
1879 }
1880
1881 static void gfx_v11_0_init_gds_vmid(struct amdgpu_device *adev)
1882 {
1883         int vmid;
1884
1885         /*
1886          * Initialize all compute and user-gfx VMIDs to have no GDS, GWS, or OA
1887          * access. Compute VMIDs should be enabled by FW for target VMIDs,
1888          * the driver can enable them for graphics. VMID0 should maintain
1889          * access so that HWS firmware can save/restore entries.
1890          */
1891         for (vmid = 1; vmid < 16; vmid++) {
1892                 WREG32_SOC15_OFFSET(GC, 0, regGDS_VMID0_BASE, 2 * vmid, 0);
1893                 WREG32_SOC15_OFFSET(GC, 0, regGDS_VMID0_SIZE, 2 * vmid, 0);
1894                 WREG32_SOC15_OFFSET(GC, 0, regGDS_GWS_VMID0, vmid, 0);
1895                 WREG32_SOC15_OFFSET(GC, 0, regGDS_OA_VMID0, vmid, 0);
1896         }
1897 }
1898
1899 static void gfx_v11_0_tcp_harvest(struct amdgpu_device *adev)
1900 {
1901         /* TODO: harvest feature to be added later. */
1902 }
1903
1904 static void gfx_v11_0_get_tcc_info(struct amdgpu_device *adev)
1905 {
1906         /* TCCs are global (not instanced). */
1907         uint32_t tcc_disable = RREG32_SOC15(GC, 0, regCGTS_TCC_DISABLE) |
1908                                RREG32_SOC15(GC, 0, regCGTS_USER_TCC_DISABLE);
1909
1910         adev->gfx.config.tcc_disabled_mask =
1911                 REG_GET_FIELD(tcc_disable, CGTS_TCC_DISABLE, TCC_DISABLE) |
1912                 (REG_GET_FIELD(tcc_disable, CGTS_TCC_DISABLE, HI_TCC_DISABLE) << 16);
1913 }
1914
1915 static void gfx_v11_0_constants_init(struct amdgpu_device *adev)
1916 {
1917         u32 tmp;
1918         int i;
1919
1920         if (!amdgpu_sriov_vf(adev))
1921                 WREG32_FIELD15_PREREG(GC, 0, GRBM_CNTL, READ_TIMEOUT, 0xff);
1922
1923         gfx_v11_0_setup_rb(adev);
1924         gfx_v11_0_get_cu_info(adev, &adev->gfx.cu_info);
1925         gfx_v11_0_get_tcc_info(adev);
1926         adev->gfx.config.pa_sc_tile_steering_override = 0;
1927
1928         /* Set whether texture coordinate truncation is conformant. */
1929         tmp = RREG32_SOC15(GC, 0, regTA_CNTL2);
1930         adev->gfx.config.ta_cntl2_truncate_coord_mode =
1931                 REG_GET_FIELD(tmp, TA_CNTL2, TRUNCATE_COORD_MODE);
1932
1933         /* XXX SH_MEM regs */
1934         /* where to put LDS, scratch, GPUVM in FSA64 space */
1935         mutex_lock(&adev->srbm_mutex);
1936         for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB(0)].num_ids; i++) {
1937                 soc21_grbm_select(adev, 0, 0, 0, i);
1938                 /* CP and shaders */
1939                 WREG32_SOC15(GC, 0, regSH_MEM_CONFIG, DEFAULT_SH_MEM_CONFIG);
1940                 if (i != 0) {
1941                         tmp = REG_SET_FIELD(0, SH_MEM_BASES, PRIVATE_BASE,
1942                                 (adev->gmc.private_aperture_start >> 48));
1943                         tmp = REG_SET_FIELD(tmp, SH_MEM_BASES, SHARED_BASE,
1944                                 (adev->gmc.shared_aperture_start >> 48));
1945                         WREG32_SOC15(GC, 0, regSH_MEM_BASES, tmp);
1946                 }
1947         }
1948         soc21_grbm_select(adev, 0, 0, 0, 0);
1949
1950         mutex_unlock(&adev->srbm_mutex);
1951
1952         gfx_v11_0_init_compute_vmid(adev);
1953         gfx_v11_0_init_gds_vmid(adev);
1954 }
1955
1956 static void gfx_v11_0_enable_gui_idle_interrupt(struct amdgpu_device *adev,
1957                                                bool enable)
1958 {
1959         u32 tmp;
1960
1961         if (amdgpu_sriov_vf(adev))
1962                 return;
1963
1964         tmp = RREG32_SOC15(GC, 0, regCP_INT_CNTL_RING0);
1965
1966         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE,
1967                             enable ? 1 : 0);
1968         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE,
1969                             enable ? 1 : 0);
1970         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE,
1971                             enable ? 1 : 0);
1972         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE,
1973                             enable ? 1 : 0);
1974
1975         WREG32_SOC15(GC, 0, regCP_INT_CNTL_RING0, tmp);
1976 }
1977
1978 static int gfx_v11_0_init_csb(struct amdgpu_device *adev)
1979 {
1980         adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr);
1981
1982         WREG32_SOC15(GC, 0, regRLC_CSIB_ADDR_HI,
1983                         adev->gfx.rlc.clear_state_gpu_addr >> 32);
1984         WREG32_SOC15(GC, 0, regRLC_CSIB_ADDR_LO,
1985                         adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc);
1986         WREG32_SOC15(GC, 0, regRLC_CSIB_LENGTH, adev->gfx.rlc.clear_state_size);
1987
1988         return 0;
1989 }
1990
1991 static void gfx_v11_0_rlc_stop(struct amdgpu_device *adev)
1992 {
1993         u32 tmp = RREG32_SOC15(GC, 0, regRLC_CNTL);
1994
1995         tmp = REG_SET_FIELD(tmp, RLC_CNTL, RLC_ENABLE_F32, 0);
1996         WREG32_SOC15(GC, 0, regRLC_CNTL, tmp);
1997 }
1998
1999 static void gfx_v11_0_rlc_reset(struct amdgpu_device *adev)
2000 {
2001         WREG32_FIELD15_PREREG(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
2002         udelay(50);
2003         WREG32_FIELD15_PREREG(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 0);
2004         udelay(50);
2005 }
2006
2007 static void gfx_v11_0_rlc_smu_handshake_cntl(struct amdgpu_device *adev,
2008                                              bool enable)
2009 {
2010         uint32_t rlc_pg_cntl;
2011
2012         rlc_pg_cntl = RREG32_SOC15(GC, 0, regRLC_PG_CNTL);
2013
2014         if (!enable) {
2015                 /* RLC_PG_CNTL[23] = 0 (default)
2016                  * RLC will wait for handshake acks with SMU
2017                  * GFXOFF will be enabled
2018                  * RLC_PG_CNTL[23] = 1
2019                  * RLC will not issue any message to SMU
2020                  * hence no handshake between SMU & RLC
2021                  * GFXOFF will be disabled
2022                  */
2023                 rlc_pg_cntl |= RLC_PG_CNTL__SMU_HANDSHAKE_DISABLE_MASK;
2024         } else
2025                 rlc_pg_cntl &= ~RLC_PG_CNTL__SMU_HANDSHAKE_DISABLE_MASK;
2026         WREG32_SOC15(GC, 0, regRLC_PG_CNTL, rlc_pg_cntl);
2027 }
2028
2029 static void gfx_v11_0_rlc_start(struct amdgpu_device *adev)
2030 {
2031         /* TODO: enable rlc & smu handshake until smu
2032          * and gfxoff feature works as expected */
2033         if (!(amdgpu_pp_feature_mask & PP_GFXOFF_MASK))
2034                 gfx_v11_0_rlc_smu_handshake_cntl(adev, false);
2035
2036         WREG32_FIELD15_PREREG(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 1);
2037         udelay(50);
2038 }
2039
2040 static void gfx_v11_0_rlc_enable_srm(struct amdgpu_device *adev)
2041 {
2042         uint32_t tmp;
2043
2044         /* enable Save Restore Machine */
2045         tmp = RREG32(SOC15_REG_OFFSET(GC, 0, regRLC_SRM_CNTL));
2046         tmp |= RLC_SRM_CNTL__AUTO_INCR_ADDR_MASK;
2047         tmp |= RLC_SRM_CNTL__SRM_ENABLE_MASK;
2048         WREG32(SOC15_REG_OFFSET(GC, 0, regRLC_SRM_CNTL), tmp);
2049 }
2050
2051 static void gfx_v11_0_load_rlcg_microcode(struct amdgpu_device *adev)
2052 {
2053         const struct rlc_firmware_header_v2_0 *hdr;
2054         const __le32 *fw_data;
2055         unsigned i, fw_size;
2056
2057         hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
2058         fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
2059                            le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2060         fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
2061
2062         WREG32_SOC15(GC, 0, regRLC_GPM_UCODE_ADDR,
2063                      RLCG_UCODE_LOADING_START_ADDRESS);
2064
2065         for (i = 0; i < fw_size; i++)
2066                 WREG32_SOC15(GC, 0, regRLC_GPM_UCODE_DATA,
2067                              le32_to_cpup(fw_data++));
2068
2069         WREG32_SOC15(GC, 0, regRLC_GPM_UCODE_ADDR, adev->gfx.rlc_fw_version);
2070 }
2071
2072 static void gfx_v11_0_load_rlc_iram_dram_microcode(struct amdgpu_device *adev)
2073 {
2074         const struct rlc_firmware_header_v2_2 *hdr;
2075         const __le32 *fw_data;
2076         unsigned i, fw_size;
2077         u32 tmp;
2078
2079         hdr = (const struct rlc_firmware_header_v2_2 *)adev->gfx.rlc_fw->data;
2080
2081         fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
2082                         le32_to_cpu(hdr->rlc_iram_ucode_offset_bytes));
2083         fw_size = le32_to_cpu(hdr->rlc_iram_ucode_size_bytes) / 4;
2084
2085         WREG32_SOC15(GC, 0, regRLC_LX6_IRAM_ADDR, 0);
2086
2087         for (i = 0; i < fw_size; i++) {
2088                 if ((amdgpu_emu_mode == 1) && (i % 100 == 99))
2089                         msleep(1);
2090                 WREG32_SOC15(GC, 0, regRLC_LX6_IRAM_DATA,
2091                                 le32_to_cpup(fw_data++));
2092         }
2093
2094         WREG32_SOC15(GC, 0, regRLC_LX6_IRAM_ADDR, adev->gfx.rlc_fw_version);
2095
2096         fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
2097                         le32_to_cpu(hdr->rlc_dram_ucode_offset_bytes));
2098         fw_size = le32_to_cpu(hdr->rlc_dram_ucode_size_bytes) / 4;
2099
2100         WREG32_SOC15(GC, 0, regRLC_LX6_DRAM_ADDR, 0);
2101         for (i = 0; i < fw_size; i++) {
2102                 if ((amdgpu_emu_mode == 1) && (i % 100 == 99))
2103                         msleep(1);
2104                 WREG32_SOC15(GC, 0, regRLC_LX6_DRAM_DATA,
2105                                 le32_to_cpup(fw_data++));
2106         }
2107
2108         WREG32_SOC15(GC, 0, regRLC_LX6_IRAM_ADDR, adev->gfx.rlc_fw_version);
2109
2110         tmp = RREG32_SOC15(GC, 0, regRLC_LX6_CNTL);
2111         tmp = REG_SET_FIELD(tmp, RLC_LX6_CNTL, PDEBUG_ENABLE, 1);
2112         tmp = REG_SET_FIELD(tmp, RLC_LX6_CNTL, BRESET, 0);
2113         WREG32_SOC15(GC, 0, regRLC_LX6_CNTL, tmp);
2114 }
2115
2116 static void gfx_v11_0_load_rlcp_rlcv_microcode(struct amdgpu_device *adev)
2117 {
2118         const struct rlc_firmware_header_v2_3 *hdr;
2119         const __le32 *fw_data;
2120         unsigned i, fw_size;
2121         u32 tmp;
2122
2123         hdr = (const struct rlc_firmware_header_v2_3 *)adev->gfx.rlc_fw->data;
2124
2125         fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
2126                         le32_to_cpu(hdr->rlcp_ucode_offset_bytes));
2127         fw_size = le32_to_cpu(hdr->rlcp_ucode_size_bytes) / 4;
2128
2129         WREG32_SOC15(GC, 0, regRLC_PACE_UCODE_ADDR, 0);
2130
2131         for (i = 0; i < fw_size; i++) {
2132                 if ((amdgpu_emu_mode == 1) && (i % 100 == 99))
2133                         msleep(1);
2134                 WREG32_SOC15(GC, 0, regRLC_PACE_UCODE_DATA,
2135                                 le32_to_cpup(fw_data++));
2136         }
2137
2138         WREG32_SOC15(GC, 0, regRLC_PACE_UCODE_ADDR, adev->gfx.rlc_fw_version);
2139
2140         tmp = RREG32_SOC15(GC, 0, regRLC_GPM_THREAD_ENABLE);
2141         tmp = REG_SET_FIELD(tmp, RLC_GPM_THREAD_ENABLE, THREAD1_ENABLE, 1);
2142         WREG32_SOC15(GC, 0, regRLC_GPM_THREAD_ENABLE, tmp);
2143
2144         fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
2145                         le32_to_cpu(hdr->rlcv_ucode_offset_bytes));
2146         fw_size = le32_to_cpu(hdr->rlcv_ucode_size_bytes) / 4;
2147
2148         WREG32_SOC15(GC, 0, regRLC_GPU_IOV_UCODE_ADDR, 0);
2149
2150         for (i = 0; i < fw_size; i++) {
2151                 if ((amdgpu_emu_mode == 1) && (i % 100 == 99))
2152                         msleep(1);
2153                 WREG32_SOC15(GC, 0, regRLC_GPU_IOV_UCODE_DATA,
2154                                 le32_to_cpup(fw_data++));
2155         }
2156
2157         WREG32_SOC15(GC, 0, regRLC_GPU_IOV_UCODE_ADDR, adev->gfx.rlc_fw_version);
2158
2159         tmp = RREG32_SOC15(GC, 0, regRLC_GPU_IOV_F32_CNTL);
2160         tmp = REG_SET_FIELD(tmp, RLC_GPU_IOV_F32_CNTL, ENABLE, 1);
2161         WREG32_SOC15(GC, 0, regRLC_GPU_IOV_F32_CNTL, tmp);
2162 }
2163
2164 static int gfx_v11_0_rlc_load_microcode(struct amdgpu_device *adev)
2165 {
2166         const struct rlc_firmware_header_v2_0 *hdr;
2167         uint16_t version_major;
2168         uint16_t version_minor;
2169
2170         if (!adev->gfx.rlc_fw)
2171                 return -EINVAL;
2172
2173         hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
2174         amdgpu_ucode_print_rlc_hdr(&hdr->header);
2175
2176         version_major = le16_to_cpu(hdr->header.header_version_major);
2177         version_minor = le16_to_cpu(hdr->header.header_version_minor);
2178
2179         if (version_major == 2) {
2180                 gfx_v11_0_load_rlcg_microcode(adev);
2181                 if (amdgpu_dpm == 1) {
2182                         if (version_minor >= 2)
2183                                 gfx_v11_0_load_rlc_iram_dram_microcode(adev);
2184                         if (version_minor == 3)
2185                                 gfx_v11_0_load_rlcp_rlcv_microcode(adev);
2186                 }
2187
2188                 return 0;
2189         }
2190
2191         return -EINVAL;
2192 }
2193
2194 static int gfx_v11_0_rlc_resume(struct amdgpu_device *adev)
2195 {
2196         int r;
2197
2198         if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
2199                 gfx_v11_0_init_csb(adev);
2200
2201                 if (!amdgpu_sriov_vf(adev)) /* enable RLC SRM */
2202                         gfx_v11_0_rlc_enable_srm(adev);
2203         } else {
2204                 if (amdgpu_sriov_vf(adev)) {
2205                         gfx_v11_0_init_csb(adev);
2206                         return 0;
2207                 }
2208
2209                 adev->gfx.rlc.funcs->stop(adev);
2210
2211                 /* disable CG */
2212                 WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL, 0);
2213
2214                 /* disable PG */
2215                 WREG32_SOC15(GC, 0, regRLC_PG_CNTL, 0);
2216
2217                 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
2218                         /* legacy rlc firmware loading */
2219                         r = gfx_v11_0_rlc_load_microcode(adev);
2220                         if (r)
2221                                 return r;
2222                 }
2223
2224                 gfx_v11_0_init_csb(adev);
2225
2226                 adev->gfx.rlc.funcs->start(adev);
2227         }
2228         return 0;
2229 }
2230
2231 static int gfx_v11_0_config_me_cache(struct amdgpu_device *adev, uint64_t addr)
2232 {
2233         uint32_t usec_timeout = 50000;  /* wait for 50ms */
2234         uint32_t tmp;
2235         int i;
2236
2237         /* Trigger an invalidation of the L1 instruction caches */
2238         tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
2239         tmp = REG_SET_FIELD(tmp, CP_ME_IC_OP_CNTL, INVALIDATE_CACHE, 1);
2240         WREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL, tmp);
2241
2242         /* Wait for invalidation complete */
2243         for (i = 0; i < usec_timeout; i++) {
2244                 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
2245                 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL,
2246                                         INVALIDATE_CACHE_COMPLETE))
2247                         break;
2248                 udelay(1);
2249         }
2250
2251         if (i >= usec_timeout) {
2252                 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2253                 return -EINVAL;
2254         }
2255
2256         if (amdgpu_emu_mode == 1)
2257                 adev->hdp.funcs->flush_hdp(adev, NULL);
2258
2259         tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL);
2260         tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, VMID, 0);
2261         tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, CACHE_POLICY, 0);
2262         tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, EXE_DISABLE, 0);
2263         tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, ADDRESS_CLAMP, 1);
2264         WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL, tmp);
2265
2266         /* Program me ucode address into intruction cache address register */
2267         WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_LO,
2268                         lower_32_bits(addr) & 0xFFFFF000);
2269         WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_HI,
2270                         upper_32_bits(addr));
2271
2272         return 0;
2273 }
2274
2275 static int gfx_v11_0_config_pfp_cache(struct amdgpu_device *adev, uint64_t addr)
2276 {
2277         uint32_t usec_timeout = 50000;  /* wait for 50ms */
2278         uint32_t tmp;
2279         int i;
2280
2281         /* Trigger an invalidation of the L1 instruction caches */
2282         tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
2283         tmp = REG_SET_FIELD(tmp, CP_PFP_IC_OP_CNTL, INVALIDATE_CACHE, 1);
2284         WREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL, tmp);
2285
2286         /* Wait for invalidation complete */
2287         for (i = 0; i < usec_timeout; i++) {
2288                 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
2289                 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL,
2290                                         INVALIDATE_CACHE_COMPLETE))
2291                         break;
2292                 udelay(1);
2293         }
2294
2295         if (i >= usec_timeout) {
2296                 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2297                 return -EINVAL;
2298         }
2299
2300         if (amdgpu_emu_mode == 1)
2301                 adev->hdp.funcs->flush_hdp(adev, NULL);
2302
2303         tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL);
2304         tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, VMID, 0);
2305         tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, CACHE_POLICY, 0);
2306         tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, EXE_DISABLE, 0);
2307         tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, ADDRESS_CLAMP, 1);
2308         WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL, tmp);
2309
2310         /* Program pfp ucode address into intruction cache address register */
2311         WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_LO,
2312                         lower_32_bits(addr) & 0xFFFFF000);
2313         WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_HI,
2314                         upper_32_bits(addr));
2315
2316         return 0;
2317 }
2318
2319 static int gfx_v11_0_config_mec_cache(struct amdgpu_device *adev, uint64_t addr)
2320 {
2321         uint32_t usec_timeout = 50000;  /* wait for 50ms */
2322         uint32_t tmp;
2323         int i;
2324
2325         /* Trigger an invalidation of the L1 instruction caches */
2326         tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL);
2327         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_OP_CNTL, INVALIDATE_CACHE, 1);
2328
2329         WREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL, tmp);
2330
2331         /* Wait for invalidation complete */
2332         for (i = 0; i < usec_timeout; i++) {
2333                 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL);
2334                 if (1 == REG_GET_FIELD(tmp, CP_CPC_IC_OP_CNTL,
2335                                         INVALIDATE_CACHE_COMPLETE))
2336                         break;
2337                 udelay(1);
2338         }
2339
2340         if (i >= usec_timeout) {
2341                 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2342                 return -EINVAL;
2343         }
2344
2345         if (amdgpu_emu_mode == 1)
2346                 adev->hdp.funcs->flush_hdp(adev, NULL);
2347
2348         tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL);
2349         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0);
2350         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, EXE_DISABLE, 0);
2351         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, ADDRESS_CLAMP, 1);
2352         WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL, tmp);
2353
2354         /* Program mec1 ucode address into intruction cache address register */
2355         WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_LO,
2356                         lower_32_bits(addr) & 0xFFFFF000);
2357         WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_HI,
2358                         upper_32_bits(addr));
2359
2360         return 0;
2361 }
2362
2363 static int gfx_v11_0_config_pfp_cache_rs64(struct amdgpu_device *adev, uint64_t addr, uint64_t addr2)
2364 {
2365         uint32_t usec_timeout = 50000;  /* wait for 50ms */
2366         uint32_t tmp;
2367         unsigned i, pipe_id;
2368         const struct gfx_firmware_header_v2_0 *pfp_hdr;
2369
2370         pfp_hdr = (const struct gfx_firmware_header_v2_0 *)
2371                 adev->gfx.pfp_fw->data;
2372
2373         WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_LO,
2374                 lower_32_bits(addr));
2375         WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_HI,
2376                 upper_32_bits(addr));
2377
2378         tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL);
2379         tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, VMID, 0);
2380         tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, CACHE_POLICY, 0);
2381         tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, EXE_DISABLE, 0);
2382         WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL, tmp);
2383
2384         /*
2385          * Programming any of the CP_PFP_IC_BASE registers
2386          * forces invalidation of the ME L1 I$. Wait for the
2387          * invalidation complete
2388          */
2389         for (i = 0; i < usec_timeout; i++) {
2390                 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
2391                 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL,
2392                         INVALIDATE_CACHE_COMPLETE))
2393                         break;
2394                 udelay(1);
2395         }
2396
2397         if (i >= usec_timeout) {
2398                 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2399                 return -EINVAL;
2400         }
2401
2402         /* Prime the L1 instruction caches */
2403         tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
2404         tmp = REG_SET_FIELD(tmp, CP_PFP_IC_OP_CNTL, PRIME_ICACHE, 1);
2405         WREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL, tmp);
2406         /* Waiting for cache primed*/
2407         for (i = 0; i < usec_timeout; i++) {
2408                 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
2409                 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL,
2410                         ICACHE_PRIMED))
2411                         break;
2412                 udelay(1);
2413         }
2414
2415         if (i >= usec_timeout) {
2416                 dev_err(adev->dev, "failed to prime instruction cache\n");
2417                 return -EINVAL;
2418         }
2419
2420         mutex_lock(&adev->srbm_mutex);
2421         for (pipe_id = 0; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) {
2422                 soc21_grbm_select(adev, 0, pipe_id, 0, 0);
2423                 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START,
2424                         (pfp_hdr->ucode_start_addr_hi << 30) |
2425                         (pfp_hdr->ucode_start_addr_lo >> 2));
2426                 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START_HI,
2427                         pfp_hdr->ucode_start_addr_hi >> 2);
2428
2429                 /*
2430                  * Program CP_ME_CNTL to reset given PIPE to take
2431                  * effect of CP_PFP_PRGRM_CNTR_START.
2432                  */
2433                 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL);
2434                 if (pipe_id == 0)
2435                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2436                                         PFP_PIPE0_RESET, 1);
2437                 else
2438                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2439                                         PFP_PIPE1_RESET, 1);
2440                 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2441
2442                 /* Clear pfp pipe0 reset bit. */
2443                 if (pipe_id == 0)
2444                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2445                                         PFP_PIPE0_RESET, 0);
2446                 else
2447                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2448                                         PFP_PIPE1_RESET, 0);
2449                 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2450
2451                 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE0_LO,
2452                         lower_32_bits(addr2));
2453                 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE0_HI,
2454                         upper_32_bits(addr2));
2455         }
2456         soc21_grbm_select(adev, 0, 0, 0, 0);
2457         mutex_unlock(&adev->srbm_mutex);
2458
2459         tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL);
2460         tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, VMID, 0);
2461         tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, CACHE_POLICY, 0);
2462         WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL, tmp);
2463
2464         /* Invalidate the data caches */
2465         tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
2466         tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, INVALIDATE_DCACHE, 1);
2467         WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL, tmp);
2468
2469         for (i = 0; i < usec_timeout; i++) {
2470                 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
2471                 if (1 == REG_GET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL,
2472                         INVALIDATE_DCACHE_COMPLETE))
2473                         break;
2474                 udelay(1);
2475         }
2476
2477         if (i >= usec_timeout) {
2478                 dev_err(adev->dev, "failed to invalidate RS64 data cache\n");
2479                 return -EINVAL;
2480         }
2481
2482         return 0;
2483 }
2484
2485 static int gfx_v11_0_config_me_cache_rs64(struct amdgpu_device *adev, uint64_t addr, uint64_t addr2)
2486 {
2487         uint32_t usec_timeout = 50000;  /* wait for 50ms */
2488         uint32_t tmp;
2489         unsigned i, pipe_id;
2490         const struct gfx_firmware_header_v2_0 *me_hdr;
2491
2492         me_hdr = (const struct gfx_firmware_header_v2_0 *)
2493                 adev->gfx.me_fw->data;
2494
2495         WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_LO,
2496                 lower_32_bits(addr));
2497         WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_HI,
2498                 upper_32_bits(addr));
2499
2500         tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL);
2501         tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, VMID, 0);
2502         tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, CACHE_POLICY, 0);
2503         tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, EXE_DISABLE, 0);
2504         WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL, tmp);
2505
2506         /*
2507          * Programming any of the CP_ME_IC_BASE registers
2508          * forces invalidation of the ME L1 I$. Wait for the
2509          * invalidation complete
2510          */
2511         for (i = 0; i < usec_timeout; i++) {
2512                 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
2513                 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL,
2514                         INVALIDATE_CACHE_COMPLETE))
2515                         break;
2516                 udelay(1);
2517         }
2518
2519         if (i >= usec_timeout) {
2520                 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2521                 return -EINVAL;
2522         }
2523
2524         /* Prime the instruction caches */
2525         tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
2526         tmp = REG_SET_FIELD(tmp, CP_ME_IC_OP_CNTL, PRIME_ICACHE, 1);
2527         WREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL, tmp);
2528
2529         /* Waiting for instruction cache primed*/
2530         for (i = 0; i < usec_timeout; i++) {
2531                 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
2532                 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL,
2533                         ICACHE_PRIMED))
2534                         break;
2535                 udelay(1);
2536         }
2537
2538         if (i >= usec_timeout) {
2539                 dev_err(adev->dev, "failed to prime instruction cache\n");
2540                 return -EINVAL;
2541         }
2542
2543         mutex_lock(&adev->srbm_mutex);
2544         for (pipe_id = 0; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) {
2545                 soc21_grbm_select(adev, 0, pipe_id, 0, 0);
2546                 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START,
2547                         (me_hdr->ucode_start_addr_hi << 30) |
2548                         (me_hdr->ucode_start_addr_lo >> 2) );
2549                 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START_HI,
2550                         me_hdr->ucode_start_addr_hi>>2);
2551
2552                 /*
2553                  * Program CP_ME_CNTL to reset given PIPE to take
2554                  * effect of CP_PFP_PRGRM_CNTR_START.
2555                  */
2556                 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL);
2557                 if (pipe_id == 0)
2558                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2559                                         ME_PIPE0_RESET, 1);
2560                 else
2561                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2562                                         ME_PIPE1_RESET, 1);
2563                 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2564
2565                 /* Clear pfp pipe0 reset bit. */
2566                 if (pipe_id == 0)
2567                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2568                                         ME_PIPE0_RESET, 0);
2569                 else
2570                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2571                                         ME_PIPE1_RESET, 0);
2572                 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2573
2574                 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE1_LO,
2575                         lower_32_bits(addr2));
2576                 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE1_HI,
2577                         upper_32_bits(addr2));
2578         }
2579         soc21_grbm_select(adev, 0, 0, 0, 0);
2580         mutex_unlock(&adev->srbm_mutex);
2581
2582         tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL);
2583         tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, VMID, 0);
2584         tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, CACHE_POLICY, 0);
2585         WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL, tmp);
2586
2587         /* Invalidate the data caches */
2588         tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
2589         tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, INVALIDATE_DCACHE, 1);
2590         WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL, tmp);
2591
2592         for (i = 0; i < usec_timeout; i++) {
2593                 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
2594                 if (1 == REG_GET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL,
2595                         INVALIDATE_DCACHE_COMPLETE))
2596                         break;
2597                 udelay(1);
2598         }
2599
2600         if (i >= usec_timeout) {
2601                 dev_err(adev->dev, "failed to invalidate RS64 data cache\n");
2602                 return -EINVAL;
2603         }
2604
2605         return 0;
2606 }
2607
2608 static int gfx_v11_0_config_mec_cache_rs64(struct amdgpu_device *adev, uint64_t addr, uint64_t addr2)
2609 {
2610         uint32_t usec_timeout = 50000;  /* wait for 50ms */
2611         uint32_t tmp;
2612         unsigned i;
2613         const struct gfx_firmware_header_v2_0 *mec_hdr;
2614
2615         mec_hdr = (const struct gfx_firmware_header_v2_0 *)
2616                 adev->gfx.mec_fw->data;
2617
2618         tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL);
2619         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0);
2620         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, EXE_DISABLE, 0);
2621         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0);
2622         WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL, tmp);
2623
2624         tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_BASE_CNTL);
2625         tmp = REG_SET_FIELD(tmp, CP_MEC_DC_BASE_CNTL, VMID, 0);
2626         tmp = REG_SET_FIELD(tmp, CP_MEC_DC_BASE_CNTL, CACHE_POLICY, 0);
2627         WREG32_SOC15(GC, 0, regCP_MEC_DC_BASE_CNTL, tmp);
2628
2629         mutex_lock(&adev->srbm_mutex);
2630         for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) {
2631                 soc21_grbm_select(adev, 1, i, 0, 0);
2632
2633                 WREG32_SOC15(GC, 0, regCP_MEC_MDBASE_LO, addr2);
2634                 WREG32_SOC15(GC, 0, regCP_MEC_MDBASE_HI,
2635                      upper_32_bits(addr2));
2636
2637                 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START,
2638                                         mec_hdr->ucode_start_addr_lo >> 2 |
2639                                         mec_hdr->ucode_start_addr_hi << 30);
2640                 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START_HI,
2641                                         mec_hdr->ucode_start_addr_hi >> 2);
2642
2643                 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_LO, addr);
2644                 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_HI,
2645                      upper_32_bits(addr));
2646         }
2647         mutex_unlock(&adev->srbm_mutex);
2648         soc21_grbm_select(adev, 0, 0, 0, 0);
2649
2650         /* Trigger an invalidation of the L1 instruction caches */
2651         tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL);
2652         tmp = REG_SET_FIELD(tmp, CP_MEC_DC_OP_CNTL, INVALIDATE_DCACHE, 1);
2653         WREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL, tmp);
2654
2655         /* Wait for invalidation complete */
2656         for (i = 0; i < usec_timeout; i++) {
2657                 tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL);
2658                 if (1 == REG_GET_FIELD(tmp, CP_MEC_DC_OP_CNTL,
2659                                        INVALIDATE_DCACHE_COMPLETE))
2660                         break;
2661                 udelay(1);
2662         }
2663
2664         if (i >= usec_timeout) {
2665                 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2666                 return -EINVAL;
2667         }
2668
2669         /* Trigger an invalidation of the L1 instruction caches */
2670         tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL);
2671         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_OP_CNTL, INVALIDATE_CACHE, 1);
2672         WREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL, tmp);
2673
2674         /* Wait for invalidation complete */
2675         for (i = 0; i < usec_timeout; i++) {
2676                 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL);
2677                 if (1 == REG_GET_FIELD(tmp, CP_CPC_IC_OP_CNTL,
2678                                        INVALIDATE_CACHE_COMPLETE))
2679                         break;
2680                 udelay(1);
2681         }
2682
2683         if (i >= usec_timeout) {
2684                 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2685                 return -EINVAL;
2686         }
2687
2688         return 0;
2689 }
2690
2691 static void gfx_v11_0_config_gfx_rs64(struct amdgpu_device *adev)
2692 {
2693         const struct gfx_firmware_header_v2_0 *pfp_hdr;
2694         const struct gfx_firmware_header_v2_0 *me_hdr;
2695         const struct gfx_firmware_header_v2_0 *mec_hdr;
2696         uint32_t pipe_id, tmp;
2697
2698         mec_hdr = (const struct gfx_firmware_header_v2_0 *)
2699                 adev->gfx.mec_fw->data;
2700         me_hdr = (const struct gfx_firmware_header_v2_0 *)
2701                 adev->gfx.me_fw->data;
2702         pfp_hdr = (const struct gfx_firmware_header_v2_0 *)
2703                 adev->gfx.pfp_fw->data;
2704
2705         /* config pfp program start addr */
2706         for (pipe_id = 0; pipe_id < 2; pipe_id++) {
2707                 soc21_grbm_select(adev, 0, pipe_id, 0, 0);
2708                 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START,
2709                         (pfp_hdr->ucode_start_addr_hi << 30) |
2710                         (pfp_hdr->ucode_start_addr_lo >> 2));
2711                 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START_HI,
2712                         pfp_hdr->ucode_start_addr_hi >> 2);
2713         }
2714         soc21_grbm_select(adev, 0, 0, 0, 0);
2715
2716         /* reset pfp pipe */
2717         tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL);
2718         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE0_RESET, 1);
2719         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE1_RESET, 1);
2720         WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2721
2722         /* clear pfp pipe reset */
2723         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE0_RESET, 0);
2724         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE1_RESET, 0);
2725         WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2726
2727         /* config me program start addr */
2728         for (pipe_id = 0; pipe_id < 2; pipe_id++) {
2729                 soc21_grbm_select(adev, 0, pipe_id, 0, 0);
2730                 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START,
2731                         (me_hdr->ucode_start_addr_hi << 30) |
2732                         (me_hdr->ucode_start_addr_lo >> 2) );
2733                 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START_HI,
2734                         me_hdr->ucode_start_addr_hi>>2);
2735         }
2736         soc21_grbm_select(adev, 0, 0, 0, 0);
2737
2738         /* reset me pipe */
2739         tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL);
2740         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE0_RESET, 1);
2741         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE1_RESET, 1);
2742         WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2743
2744         /* clear me pipe reset */
2745         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE0_RESET, 0);
2746         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE1_RESET, 0);
2747         WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2748
2749         /* config mec program start addr */
2750         for (pipe_id = 0; pipe_id < 4; pipe_id++) {
2751                 soc21_grbm_select(adev, 1, pipe_id, 0, 0);
2752                 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START,
2753                                         mec_hdr->ucode_start_addr_lo >> 2 |
2754                                         mec_hdr->ucode_start_addr_hi << 30);
2755                 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START_HI,
2756                                         mec_hdr->ucode_start_addr_hi >> 2);
2757         }
2758         soc21_grbm_select(adev, 0, 0, 0, 0);
2759
2760         /* reset mec pipe */
2761         tmp = RREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL);
2762         tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE0_RESET, 1);
2763         tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE1_RESET, 1);
2764         tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE2_RESET, 1);
2765         tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE3_RESET, 1);
2766         WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, tmp);
2767
2768         /* clear mec pipe reset */
2769         tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE0_RESET, 0);
2770         tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE1_RESET, 0);
2771         tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE2_RESET, 0);
2772         tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE3_RESET, 0);
2773         WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, tmp);
2774 }
2775
2776 static int gfx_v11_0_wait_for_rlc_autoload_complete(struct amdgpu_device *adev)
2777 {
2778         uint32_t cp_status;
2779         uint32_t bootload_status;
2780         int i, r;
2781         uint64_t addr, addr2;
2782
2783         for (i = 0; i < adev->usec_timeout; i++) {
2784                 cp_status = RREG32_SOC15(GC, 0, regCP_STAT);
2785
2786                 if (amdgpu_ip_version(adev, GC_HWIP, 0) ==
2787                             IP_VERSION(11, 0, 1) ||
2788                     amdgpu_ip_version(adev, GC_HWIP, 0) ==
2789                             IP_VERSION(11, 0, 4) ||
2790                     amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 5, 0) ||
2791                     amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 5, 1) ||
2792                     amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 5, 2))
2793                         bootload_status = RREG32_SOC15(GC, 0,
2794                                         regRLC_RLCS_BOOTLOAD_STATUS_gc_11_0_1);
2795                 else
2796                         bootload_status = RREG32_SOC15(GC, 0, regRLC_RLCS_BOOTLOAD_STATUS);
2797
2798                 if ((cp_status == 0) &&
2799                     (REG_GET_FIELD(bootload_status,
2800                         RLC_RLCS_BOOTLOAD_STATUS, BOOTLOAD_COMPLETE) == 1)) {
2801                         break;
2802                 }
2803                 udelay(1);
2804         }
2805
2806         if (i >= adev->usec_timeout) {
2807                 dev_err(adev->dev, "rlc autoload: gc ucode autoload timeout\n");
2808                 return -ETIMEDOUT;
2809         }
2810
2811         if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) {
2812                 if (adev->gfx.rs64_enable) {
2813                         addr = adev->gfx.rlc.rlc_autoload_gpu_addr +
2814                                 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_ME].offset;
2815                         addr2 = adev->gfx.rlc.rlc_autoload_gpu_addr +
2816                                 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_ME_P0_STACK].offset;
2817                         r = gfx_v11_0_config_me_cache_rs64(adev, addr, addr2);
2818                         if (r)
2819                                 return r;
2820                         addr = adev->gfx.rlc.rlc_autoload_gpu_addr +
2821                                 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_PFP].offset;
2822                         addr2 = adev->gfx.rlc.rlc_autoload_gpu_addr +
2823                                 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_PFP_P0_STACK].offset;
2824                         r = gfx_v11_0_config_pfp_cache_rs64(adev, addr, addr2);
2825                         if (r)
2826                                 return r;
2827                         addr = adev->gfx.rlc.rlc_autoload_gpu_addr +
2828                                 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_MEC].offset;
2829                         addr2 = adev->gfx.rlc.rlc_autoload_gpu_addr +
2830                                 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_MEC_P0_STACK].offset;
2831                         r = gfx_v11_0_config_mec_cache_rs64(adev, addr, addr2);
2832                         if (r)
2833                                 return r;
2834                 } else {
2835                         addr = adev->gfx.rlc.rlc_autoload_gpu_addr +
2836                                 rlc_autoload_info[SOC21_FIRMWARE_ID_CP_ME].offset;
2837                         r = gfx_v11_0_config_me_cache(adev, addr);
2838                         if (r)
2839                                 return r;
2840                         addr = adev->gfx.rlc.rlc_autoload_gpu_addr +
2841                                 rlc_autoload_info[SOC21_FIRMWARE_ID_CP_PFP].offset;
2842                         r = gfx_v11_0_config_pfp_cache(adev, addr);
2843                         if (r)
2844                                 return r;
2845                         addr = adev->gfx.rlc.rlc_autoload_gpu_addr +
2846                                 rlc_autoload_info[SOC21_FIRMWARE_ID_CP_MEC].offset;
2847                         r = gfx_v11_0_config_mec_cache(adev, addr);
2848                         if (r)
2849                                 return r;
2850                 }
2851         }
2852
2853         return 0;
2854 }
2855
2856 static int gfx_v11_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable)
2857 {
2858         int i;
2859         u32 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL);
2860
2861         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, enable ? 0 : 1);
2862         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, enable ? 0 : 1);
2863         WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2864
2865         for (i = 0; i < adev->usec_timeout; i++) {
2866                 if (RREG32_SOC15(GC, 0, regCP_STAT) == 0)
2867                         break;
2868                 udelay(1);
2869         }
2870
2871         if (i >= adev->usec_timeout)
2872                 DRM_ERROR("failed to %s cp gfx\n", enable ? "unhalt" : "halt");
2873
2874         return 0;
2875 }
2876
2877 static int gfx_v11_0_cp_gfx_load_pfp_microcode(struct amdgpu_device *adev)
2878 {
2879         int r;
2880         const struct gfx_firmware_header_v1_0 *pfp_hdr;
2881         const __le32 *fw_data;
2882         unsigned i, fw_size;
2883
2884         pfp_hdr = (const struct gfx_firmware_header_v1_0 *)
2885                 adev->gfx.pfp_fw->data;
2886
2887         amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header);
2888
2889         fw_data = (const __le32 *)(adev->gfx.pfp_fw->data +
2890                 le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
2891         fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes);
2892
2893         r = amdgpu_bo_create_reserved(adev, pfp_hdr->header.ucode_size_bytes,
2894                                       PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
2895                                       &adev->gfx.pfp.pfp_fw_obj,
2896                                       &adev->gfx.pfp.pfp_fw_gpu_addr,
2897                                       (void **)&adev->gfx.pfp.pfp_fw_ptr);
2898         if (r) {
2899                 dev_err(adev->dev, "(%d) failed to create pfp fw bo\n", r);
2900                 gfx_v11_0_pfp_fini(adev);
2901                 return r;
2902         }
2903
2904         memcpy(adev->gfx.pfp.pfp_fw_ptr, fw_data, fw_size);
2905
2906         amdgpu_bo_kunmap(adev->gfx.pfp.pfp_fw_obj);
2907         amdgpu_bo_unreserve(adev->gfx.pfp.pfp_fw_obj);
2908
2909         gfx_v11_0_config_pfp_cache(adev, adev->gfx.pfp.pfp_fw_gpu_addr);
2910
2911         WREG32_SOC15(GC, 0, regCP_HYP_PFP_UCODE_ADDR, 0);
2912
2913         for (i = 0; i < pfp_hdr->jt_size; i++)
2914                 WREG32_SOC15(GC, 0, regCP_HYP_PFP_UCODE_DATA,
2915                              le32_to_cpup(fw_data + pfp_hdr->jt_offset + i));
2916
2917         WREG32_SOC15(GC, 0, regCP_HYP_PFP_UCODE_ADDR, adev->gfx.pfp_fw_version);
2918
2919         return 0;
2920 }
2921
2922 static int gfx_v11_0_cp_gfx_load_pfp_microcode_rs64(struct amdgpu_device *adev)
2923 {
2924         int r;
2925         const struct gfx_firmware_header_v2_0 *pfp_hdr;
2926         const __le32 *fw_ucode, *fw_data;
2927         unsigned i, pipe_id, fw_ucode_size, fw_data_size;
2928         uint32_t tmp;
2929         uint32_t usec_timeout = 50000;  /* wait for 50ms */
2930
2931         pfp_hdr = (const struct gfx_firmware_header_v2_0 *)
2932                 adev->gfx.pfp_fw->data;
2933
2934         amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header);
2935
2936         /* instruction */
2937         fw_ucode = (const __le32 *)(adev->gfx.pfp_fw->data +
2938                 le32_to_cpu(pfp_hdr->ucode_offset_bytes));
2939         fw_ucode_size = le32_to_cpu(pfp_hdr->ucode_size_bytes);
2940         /* data */
2941         fw_data = (const __le32 *)(adev->gfx.pfp_fw->data +
2942                 le32_to_cpu(pfp_hdr->data_offset_bytes));
2943         fw_data_size = le32_to_cpu(pfp_hdr->data_size_bytes);
2944
2945         /* 64kb align */
2946         r = amdgpu_bo_create_reserved(adev, fw_ucode_size,
2947                                       64 * 1024,
2948                                       AMDGPU_GEM_DOMAIN_VRAM |
2949                                       AMDGPU_GEM_DOMAIN_GTT,
2950                                       &adev->gfx.pfp.pfp_fw_obj,
2951                                       &adev->gfx.pfp.pfp_fw_gpu_addr,
2952                                       (void **)&adev->gfx.pfp.pfp_fw_ptr);
2953         if (r) {
2954                 dev_err(adev->dev, "(%d) failed to create pfp ucode fw bo\n", r);
2955                 gfx_v11_0_pfp_fini(adev);
2956                 return r;
2957         }
2958
2959         r = amdgpu_bo_create_reserved(adev, fw_data_size,
2960                                       64 * 1024,
2961                                       AMDGPU_GEM_DOMAIN_VRAM |
2962                                       AMDGPU_GEM_DOMAIN_GTT,
2963                                       &adev->gfx.pfp.pfp_fw_data_obj,
2964                                       &adev->gfx.pfp.pfp_fw_data_gpu_addr,
2965                                       (void **)&adev->gfx.pfp.pfp_fw_data_ptr);
2966         if (r) {
2967                 dev_err(adev->dev, "(%d) failed to create pfp data fw bo\n", r);
2968                 gfx_v11_0_pfp_fini(adev);
2969                 return r;
2970         }
2971
2972         memcpy(adev->gfx.pfp.pfp_fw_ptr, fw_ucode, fw_ucode_size);
2973         memcpy(adev->gfx.pfp.pfp_fw_data_ptr, fw_data, fw_data_size);
2974
2975         amdgpu_bo_kunmap(adev->gfx.pfp.pfp_fw_obj);
2976         amdgpu_bo_kunmap(adev->gfx.pfp.pfp_fw_data_obj);
2977         amdgpu_bo_unreserve(adev->gfx.pfp.pfp_fw_obj);
2978         amdgpu_bo_unreserve(adev->gfx.pfp.pfp_fw_data_obj);
2979
2980         if (amdgpu_emu_mode == 1)
2981                 adev->hdp.funcs->flush_hdp(adev, NULL);
2982
2983         WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_LO,
2984                 lower_32_bits(adev->gfx.pfp.pfp_fw_gpu_addr));
2985         WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_HI,
2986                 upper_32_bits(adev->gfx.pfp.pfp_fw_gpu_addr));
2987
2988         tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL);
2989         tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, VMID, 0);
2990         tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, CACHE_POLICY, 0);
2991         tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, EXE_DISABLE, 0);
2992         WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL, tmp);
2993
2994         /*
2995          * Programming any of the CP_PFP_IC_BASE registers
2996          * forces invalidation of the ME L1 I$. Wait for the
2997          * invalidation complete
2998          */
2999         for (i = 0; i < usec_timeout; i++) {
3000                 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
3001                 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL,
3002                         INVALIDATE_CACHE_COMPLETE))
3003                         break;
3004                 udelay(1);
3005         }
3006
3007         if (i >= usec_timeout) {
3008                 dev_err(adev->dev, "failed to invalidate instruction cache\n");
3009                 return -EINVAL;
3010         }
3011
3012         /* Prime the L1 instruction caches */
3013         tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
3014         tmp = REG_SET_FIELD(tmp, CP_PFP_IC_OP_CNTL, PRIME_ICACHE, 1);
3015         WREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL, tmp);
3016         /* Waiting for cache primed*/
3017         for (i = 0; i < usec_timeout; i++) {
3018                 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
3019                 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL,
3020                         ICACHE_PRIMED))
3021                         break;
3022                 udelay(1);
3023         }
3024
3025         if (i >= usec_timeout) {
3026                 dev_err(adev->dev, "failed to prime instruction cache\n");
3027                 return -EINVAL;
3028         }
3029
3030         mutex_lock(&adev->srbm_mutex);
3031         for (pipe_id = 0; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) {
3032                 soc21_grbm_select(adev, 0, pipe_id, 0, 0);
3033                 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START,
3034                         (pfp_hdr->ucode_start_addr_hi << 30) |
3035                         (pfp_hdr->ucode_start_addr_lo >> 2) );
3036                 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START_HI,
3037                         pfp_hdr->ucode_start_addr_hi>>2);
3038
3039                 /*
3040                  * Program CP_ME_CNTL to reset given PIPE to take
3041                  * effect of CP_PFP_PRGRM_CNTR_START.
3042                  */
3043                 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL);
3044                 if (pipe_id == 0)
3045                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
3046                                         PFP_PIPE0_RESET, 1);
3047                 else
3048                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
3049                                         PFP_PIPE1_RESET, 1);
3050                 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
3051
3052                 /* Clear pfp pipe0 reset bit. */
3053                 if (pipe_id == 0)
3054                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
3055                                         PFP_PIPE0_RESET, 0);
3056                 else
3057                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
3058                                         PFP_PIPE1_RESET, 0);
3059                 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
3060
3061                 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE0_LO,
3062                         lower_32_bits(adev->gfx.pfp.pfp_fw_data_gpu_addr));
3063                 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE0_HI,
3064                         upper_32_bits(adev->gfx.pfp.pfp_fw_data_gpu_addr));
3065         }
3066         soc21_grbm_select(adev, 0, 0, 0, 0);
3067         mutex_unlock(&adev->srbm_mutex);
3068
3069         tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL);
3070         tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, VMID, 0);
3071         tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, CACHE_POLICY, 0);
3072         WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL, tmp);
3073
3074         /* Invalidate the data caches */
3075         tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
3076         tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, INVALIDATE_DCACHE, 1);
3077         WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL, tmp);
3078
3079         for (i = 0; i < usec_timeout; i++) {
3080                 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
3081                 if (1 == REG_GET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL,
3082                         INVALIDATE_DCACHE_COMPLETE))
3083                         break;
3084                 udelay(1);
3085         }
3086
3087         if (i >= usec_timeout) {
3088                 dev_err(adev->dev, "failed to invalidate RS64 data cache\n");
3089                 return -EINVAL;
3090         }
3091
3092         return 0;
3093 }
3094
3095 static int gfx_v11_0_cp_gfx_load_me_microcode(struct amdgpu_device *adev)
3096 {
3097         int r;
3098         const struct gfx_firmware_header_v1_0 *me_hdr;
3099         const __le32 *fw_data;
3100         unsigned i, fw_size;
3101
3102         me_hdr = (const struct gfx_firmware_header_v1_0 *)
3103                 adev->gfx.me_fw->data;
3104
3105         amdgpu_ucode_print_gfx_hdr(&me_hdr->header);
3106
3107         fw_data = (const __le32 *)(adev->gfx.me_fw->data +
3108                 le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
3109         fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes);
3110
3111         r = amdgpu_bo_create_reserved(adev, me_hdr->header.ucode_size_bytes,
3112                                       PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
3113                                       &adev->gfx.me.me_fw_obj,
3114                                       &adev->gfx.me.me_fw_gpu_addr,
3115                                       (void **)&adev->gfx.me.me_fw_ptr);
3116         if (r) {
3117                 dev_err(adev->dev, "(%d) failed to create me fw bo\n", r);
3118                 gfx_v11_0_me_fini(adev);
3119                 return r;
3120         }
3121
3122         memcpy(adev->gfx.me.me_fw_ptr, fw_data, fw_size);
3123
3124         amdgpu_bo_kunmap(adev->gfx.me.me_fw_obj);
3125         amdgpu_bo_unreserve(adev->gfx.me.me_fw_obj);
3126
3127         gfx_v11_0_config_me_cache(adev, adev->gfx.me.me_fw_gpu_addr);
3128
3129         WREG32_SOC15(GC, 0, regCP_HYP_ME_UCODE_ADDR, 0);
3130
3131         for (i = 0; i < me_hdr->jt_size; i++)
3132                 WREG32_SOC15(GC, 0, regCP_HYP_ME_UCODE_DATA,
3133                              le32_to_cpup(fw_data + me_hdr->jt_offset + i));
3134
3135         WREG32_SOC15(GC, 0, regCP_HYP_ME_UCODE_ADDR, adev->gfx.me_fw_version);
3136
3137         return 0;
3138 }
3139
3140 static int gfx_v11_0_cp_gfx_load_me_microcode_rs64(struct amdgpu_device *adev)
3141 {
3142         int r;
3143         const struct gfx_firmware_header_v2_0 *me_hdr;
3144         const __le32 *fw_ucode, *fw_data;
3145         unsigned i, pipe_id, fw_ucode_size, fw_data_size;
3146         uint32_t tmp;
3147         uint32_t usec_timeout = 50000;  /* wait for 50ms */
3148
3149         me_hdr = (const struct gfx_firmware_header_v2_0 *)
3150                 adev->gfx.me_fw->data;
3151
3152         amdgpu_ucode_print_gfx_hdr(&me_hdr->header);
3153
3154         /* instruction */
3155         fw_ucode = (const __le32 *)(adev->gfx.me_fw->data +
3156                 le32_to_cpu(me_hdr->ucode_offset_bytes));
3157         fw_ucode_size = le32_to_cpu(me_hdr->ucode_size_bytes);
3158         /* data */
3159         fw_data = (const __le32 *)(adev->gfx.me_fw->data +
3160                 le32_to_cpu(me_hdr->data_offset_bytes));
3161         fw_data_size = le32_to_cpu(me_hdr->data_size_bytes);
3162
3163         /* 64kb align*/
3164         r = amdgpu_bo_create_reserved(adev, fw_ucode_size,
3165                                       64 * 1024,
3166                                       AMDGPU_GEM_DOMAIN_VRAM |
3167                                       AMDGPU_GEM_DOMAIN_GTT,
3168                                       &adev->gfx.me.me_fw_obj,
3169                                       &adev->gfx.me.me_fw_gpu_addr,
3170                                       (void **)&adev->gfx.me.me_fw_ptr);
3171         if (r) {
3172                 dev_err(adev->dev, "(%d) failed to create me ucode bo\n", r);
3173                 gfx_v11_0_me_fini(adev);
3174                 return r;
3175         }
3176
3177         r = amdgpu_bo_create_reserved(adev, fw_data_size,
3178                                       64 * 1024,
3179                                       AMDGPU_GEM_DOMAIN_VRAM |
3180                                       AMDGPU_GEM_DOMAIN_GTT,
3181                                       &adev->gfx.me.me_fw_data_obj,
3182                                       &adev->gfx.me.me_fw_data_gpu_addr,
3183                                       (void **)&adev->gfx.me.me_fw_data_ptr);
3184         if (r) {
3185                 dev_err(adev->dev, "(%d) failed to create me data bo\n", r);
3186                 gfx_v11_0_pfp_fini(adev);
3187                 return r;
3188         }
3189
3190         memcpy(adev->gfx.me.me_fw_ptr, fw_ucode, fw_ucode_size);
3191         memcpy(adev->gfx.me.me_fw_data_ptr, fw_data, fw_data_size);
3192
3193         amdgpu_bo_kunmap(adev->gfx.me.me_fw_obj);
3194         amdgpu_bo_kunmap(adev->gfx.me.me_fw_data_obj);
3195         amdgpu_bo_unreserve(adev->gfx.me.me_fw_obj);
3196         amdgpu_bo_unreserve(adev->gfx.me.me_fw_data_obj);
3197
3198         if (amdgpu_emu_mode == 1)
3199                 adev->hdp.funcs->flush_hdp(adev, NULL);
3200
3201         WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_LO,
3202                 lower_32_bits(adev->gfx.me.me_fw_gpu_addr));
3203         WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_HI,
3204                 upper_32_bits(adev->gfx.me.me_fw_gpu_addr));
3205
3206         tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL);
3207         tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, VMID, 0);
3208         tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, CACHE_POLICY, 0);
3209         tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, EXE_DISABLE, 0);
3210         WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL, tmp);
3211
3212         /*
3213          * Programming any of the CP_ME_IC_BASE registers
3214          * forces invalidation of the ME L1 I$. Wait for the
3215          * invalidation complete
3216          */
3217         for (i = 0; i < usec_timeout; i++) {
3218                 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
3219                 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL,
3220                         INVALIDATE_CACHE_COMPLETE))
3221                         break;
3222                 udelay(1);
3223         }
3224
3225         if (i >= usec_timeout) {
3226                 dev_err(adev->dev, "failed to invalidate instruction cache\n");
3227                 return -EINVAL;
3228         }
3229
3230         /* Prime the instruction caches */
3231         tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
3232         tmp = REG_SET_FIELD(tmp, CP_ME_IC_OP_CNTL, PRIME_ICACHE, 1);
3233         WREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL, tmp);
3234
3235         /* Waiting for instruction cache primed*/
3236         for (i = 0; i < usec_timeout; i++) {
3237                 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
3238                 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL,
3239                         ICACHE_PRIMED))
3240                         break;
3241                 udelay(1);
3242         }
3243
3244         if (i >= usec_timeout) {
3245                 dev_err(adev->dev, "failed to prime instruction cache\n");
3246                 return -EINVAL;
3247         }
3248
3249         mutex_lock(&adev->srbm_mutex);
3250         for (pipe_id = 0; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) {
3251                 soc21_grbm_select(adev, 0, pipe_id, 0, 0);
3252                 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START,
3253                         (me_hdr->ucode_start_addr_hi << 30) |
3254                         (me_hdr->ucode_start_addr_lo >> 2) );
3255                 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START_HI,
3256                         me_hdr->ucode_start_addr_hi>>2);
3257
3258                 /*
3259                  * Program CP_ME_CNTL to reset given PIPE to take
3260                  * effect of CP_PFP_PRGRM_CNTR_START.
3261                  */
3262                 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL);
3263                 if (pipe_id == 0)
3264                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
3265                                         ME_PIPE0_RESET, 1);
3266                 else
3267                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
3268                                         ME_PIPE1_RESET, 1);
3269                 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
3270
3271                 /* Clear pfp pipe0 reset bit. */
3272                 if (pipe_id == 0)
3273                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
3274                                         ME_PIPE0_RESET, 0);
3275                 else
3276                         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
3277                                         ME_PIPE1_RESET, 0);
3278                 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
3279
3280                 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE1_LO,
3281                         lower_32_bits(adev->gfx.me.me_fw_data_gpu_addr));
3282                 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE1_HI,
3283                         upper_32_bits(adev->gfx.me.me_fw_data_gpu_addr));
3284         }
3285         soc21_grbm_select(adev, 0, 0, 0, 0);
3286         mutex_unlock(&adev->srbm_mutex);
3287
3288         tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL);
3289         tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, VMID, 0);
3290         tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, CACHE_POLICY, 0);
3291         WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL, tmp);
3292
3293         /* Invalidate the data caches */
3294         tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
3295         tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, INVALIDATE_DCACHE, 1);
3296         WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL, tmp);
3297
3298         for (i = 0; i < usec_timeout; i++) {
3299                 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
3300                 if (1 == REG_GET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL,
3301                         INVALIDATE_DCACHE_COMPLETE))
3302                         break;
3303                 udelay(1);
3304         }
3305
3306         if (i >= usec_timeout) {
3307                 dev_err(adev->dev, "failed to invalidate RS64 data cache\n");
3308                 return -EINVAL;
3309         }
3310
3311         return 0;
3312 }
3313
3314 static int gfx_v11_0_cp_gfx_load_microcode(struct amdgpu_device *adev)
3315 {
3316         int r;
3317
3318         if (!adev->gfx.me_fw || !adev->gfx.pfp_fw)
3319                 return -EINVAL;
3320
3321         gfx_v11_0_cp_gfx_enable(adev, false);
3322
3323         if (adev->gfx.rs64_enable)
3324                 r = gfx_v11_0_cp_gfx_load_pfp_microcode_rs64(adev);
3325         else
3326                 r = gfx_v11_0_cp_gfx_load_pfp_microcode(adev);
3327         if (r) {
3328                 dev_err(adev->dev, "(%d) failed to load pfp fw\n", r);
3329                 return r;
3330         }
3331
3332         if (adev->gfx.rs64_enable)
3333                 r = gfx_v11_0_cp_gfx_load_me_microcode_rs64(adev);
3334         else
3335                 r = gfx_v11_0_cp_gfx_load_me_microcode(adev);
3336         if (r) {
3337                 dev_err(adev->dev, "(%d) failed to load me fw\n", r);
3338                 return r;
3339         }
3340
3341         return 0;
3342 }
3343
3344 static int gfx_v11_0_cp_gfx_start(struct amdgpu_device *adev)
3345 {
3346         struct amdgpu_ring *ring;
3347         const struct cs_section_def *sect = NULL;
3348         const struct cs_extent_def *ext = NULL;
3349         int r, i;
3350         int ctx_reg_offset;
3351
3352         /* init the CP */
3353         WREG32_SOC15(GC, 0, regCP_MAX_CONTEXT,
3354                      adev->gfx.config.max_hw_contexts - 1);
3355         WREG32_SOC15(GC, 0, regCP_DEVICE_ID, 1);
3356
3357         if (!amdgpu_async_gfx_ring)
3358                 gfx_v11_0_cp_gfx_enable(adev, true);
3359
3360         ring = &adev->gfx.gfx_ring[0];
3361         r = amdgpu_ring_alloc(ring, gfx_v11_0_get_csb_size(adev));
3362         if (r) {
3363                 DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r);
3364                 return r;
3365         }
3366
3367         amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3368         amdgpu_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3369
3370         amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
3371         amdgpu_ring_write(ring, 0x80000000);
3372         amdgpu_ring_write(ring, 0x80000000);
3373
3374         for (sect = gfx11_cs_data; sect->section != NULL; ++sect) {
3375                 for (ext = sect->section; ext->extent != NULL; ++ext) {
3376                         if (sect->id == SECT_CONTEXT) {
3377                                 amdgpu_ring_write(ring,
3378                                                   PACKET3(PACKET3_SET_CONTEXT_REG,
3379                                                           ext->reg_count));
3380                                 amdgpu_ring_write(ring, ext->reg_index -
3381                                                   PACKET3_SET_CONTEXT_REG_START);
3382                                 for (i = 0; i < ext->reg_count; i++)
3383                                         amdgpu_ring_write(ring, ext->extent[i]);
3384                         }
3385                 }
3386         }
3387
3388         ctx_reg_offset =
3389                 SOC15_REG_OFFSET(GC, 0, regPA_SC_TILE_STEERING_OVERRIDE) - PACKET3_SET_CONTEXT_REG_START;
3390         amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 1));
3391         amdgpu_ring_write(ring, ctx_reg_offset);
3392         amdgpu_ring_write(ring, adev->gfx.config.pa_sc_tile_steering_override);
3393
3394         amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3395         amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3396
3397         amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3398         amdgpu_ring_write(ring, 0);
3399
3400         amdgpu_ring_commit(ring);
3401
3402         /* submit cs packet to copy state 0 to next available state */
3403         if (adev->gfx.num_gfx_rings > 1) {
3404                 /* maximum supported gfx ring is 2 */
3405                 ring = &adev->gfx.gfx_ring[1];
3406                 r = amdgpu_ring_alloc(ring, 2);
3407                 if (r) {
3408                         DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r);
3409                         return r;
3410                 }
3411
3412                 amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3413                 amdgpu_ring_write(ring, 0);
3414
3415                 amdgpu_ring_commit(ring);
3416         }
3417         return 0;
3418 }
3419
3420 static void gfx_v11_0_cp_gfx_switch_pipe(struct amdgpu_device *adev,
3421                                          CP_PIPE_ID pipe)
3422 {
3423         u32 tmp;
3424
3425         tmp = RREG32_SOC15(GC, 0, regGRBM_GFX_CNTL);
3426         tmp = REG_SET_FIELD(tmp, GRBM_GFX_CNTL, PIPEID, pipe);
3427
3428         WREG32_SOC15(GC, 0, regGRBM_GFX_CNTL, tmp);
3429 }
3430
3431 static void gfx_v11_0_cp_gfx_set_doorbell(struct amdgpu_device *adev,
3432                                           struct amdgpu_ring *ring)
3433 {
3434         u32 tmp;
3435
3436         tmp = RREG32_SOC15(GC, 0, regCP_RB_DOORBELL_CONTROL);
3437         if (ring->use_doorbell) {
3438                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3439                                     DOORBELL_OFFSET, ring->doorbell_index);
3440                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3441                                     DOORBELL_EN, 1);
3442         } else {
3443                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3444                                     DOORBELL_EN, 0);
3445         }
3446         WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_CONTROL, tmp);
3447
3448         tmp = REG_SET_FIELD(0, CP_RB_DOORBELL_RANGE_LOWER,
3449                             DOORBELL_RANGE_LOWER, ring->doorbell_index);
3450         WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_RANGE_LOWER, tmp);
3451
3452         WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_RANGE_UPPER,
3453                      CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK);
3454 }
3455
3456 static int gfx_v11_0_cp_gfx_resume(struct amdgpu_device *adev)
3457 {
3458         struct amdgpu_ring *ring;
3459         u32 tmp;
3460         u32 rb_bufsz;
3461         u64 rb_addr, rptr_addr, wptr_gpu_addr;
3462
3463         /* Set the write pointer delay */
3464         WREG32_SOC15(GC, 0, regCP_RB_WPTR_DELAY, 0);
3465
3466         /* set the RB to use vmid 0 */
3467         WREG32_SOC15(GC, 0, regCP_RB_VMID, 0);
3468
3469         /* Init gfx ring 0 for pipe 0 */
3470         mutex_lock(&adev->srbm_mutex);
3471         gfx_v11_0_cp_gfx_switch_pipe(adev, PIPE_ID0);
3472
3473         /* Set ring buffer size */
3474         ring = &adev->gfx.gfx_ring[0];
3475         rb_bufsz = order_base_2(ring->ring_size / 8);
3476         tmp = REG_SET_FIELD(0, CP_RB0_CNTL, RB_BUFSZ, rb_bufsz);
3477         tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, RB_BLKSZ, rb_bufsz - 2);
3478         WREG32_SOC15(GC, 0, regCP_RB0_CNTL, tmp);
3479
3480         /* Initialize the ring buffer's write pointers */
3481         ring->wptr = 0;
3482         WREG32_SOC15(GC, 0, regCP_RB0_WPTR, lower_32_bits(ring->wptr));
3483         WREG32_SOC15(GC, 0, regCP_RB0_WPTR_HI, upper_32_bits(ring->wptr));
3484
3485         /* set the wb address wether it's enabled or not */
3486         rptr_addr = ring->rptr_gpu_addr;
3487         WREG32_SOC15(GC, 0, regCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr));
3488         WREG32_SOC15(GC, 0, regCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) &
3489                      CP_RB_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK);
3490
3491         wptr_gpu_addr = ring->wptr_gpu_addr;
3492         WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_ADDR_LO,
3493                      lower_32_bits(wptr_gpu_addr));
3494         WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_ADDR_HI,
3495                      upper_32_bits(wptr_gpu_addr));
3496
3497         mdelay(1);
3498         WREG32_SOC15(GC, 0, regCP_RB0_CNTL, tmp);
3499
3500         rb_addr = ring->gpu_addr >> 8;
3501         WREG32_SOC15(GC, 0, regCP_RB0_BASE, rb_addr);
3502         WREG32_SOC15(GC, 0, regCP_RB0_BASE_HI, upper_32_bits(rb_addr));
3503
3504         WREG32_SOC15(GC, 0, regCP_RB_ACTIVE, 1);
3505
3506         gfx_v11_0_cp_gfx_set_doorbell(adev, ring);
3507         mutex_unlock(&adev->srbm_mutex);
3508
3509         /* Init gfx ring 1 for pipe 1 */
3510         if (adev->gfx.num_gfx_rings > 1) {
3511                 mutex_lock(&adev->srbm_mutex);
3512                 gfx_v11_0_cp_gfx_switch_pipe(adev, PIPE_ID1);
3513                 /* maximum supported gfx ring is 2 */
3514                 ring = &adev->gfx.gfx_ring[1];
3515                 rb_bufsz = order_base_2(ring->ring_size / 8);
3516                 tmp = REG_SET_FIELD(0, CP_RB1_CNTL, RB_BUFSZ, rb_bufsz);
3517                 tmp = REG_SET_FIELD(tmp, CP_RB1_CNTL, RB_BLKSZ, rb_bufsz - 2);
3518                 WREG32_SOC15(GC, 0, regCP_RB1_CNTL, tmp);
3519                 /* Initialize the ring buffer's write pointers */
3520                 ring->wptr = 0;
3521                 WREG32_SOC15(GC, 0, regCP_RB1_WPTR, lower_32_bits(ring->wptr));
3522                 WREG32_SOC15(GC, 0, regCP_RB1_WPTR_HI, upper_32_bits(ring->wptr));
3523                 /* Set the wb address wether it's enabled or not */
3524                 rptr_addr = ring->rptr_gpu_addr;
3525                 WREG32_SOC15(GC, 0, regCP_RB1_RPTR_ADDR, lower_32_bits(rptr_addr));
3526                 WREG32_SOC15(GC, 0, regCP_RB1_RPTR_ADDR_HI, upper_32_bits(rptr_addr) &
3527                              CP_RB1_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK);
3528                 wptr_gpu_addr = ring->wptr_gpu_addr;
3529                 WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_ADDR_LO,
3530                              lower_32_bits(wptr_gpu_addr));
3531                 WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_ADDR_HI,
3532                              upper_32_bits(wptr_gpu_addr));
3533
3534                 mdelay(1);
3535                 WREG32_SOC15(GC, 0, regCP_RB1_CNTL, tmp);
3536
3537                 rb_addr = ring->gpu_addr >> 8;
3538                 WREG32_SOC15(GC, 0, regCP_RB1_BASE, rb_addr);
3539                 WREG32_SOC15(GC, 0, regCP_RB1_BASE_HI, upper_32_bits(rb_addr));
3540                 WREG32_SOC15(GC, 0, regCP_RB1_ACTIVE, 1);
3541
3542                 gfx_v11_0_cp_gfx_set_doorbell(adev, ring);
3543                 mutex_unlock(&adev->srbm_mutex);
3544         }
3545         /* Switch to pipe 0 */
3546         mutex_lock(&adev->srbm_mutex);
3547         gfx_v11_0_cp_gfx_switch_pipe(adev, PIPE_ID0);
3548         mutex_unlock(&adev->srbm_mutex);
3549
3550         /* start the ring */
3551         gfx_v11_0_cp_gfx_start(adev);
3552
3553         return 0;
3554 }
3555
3556 static void gfx_v11_0_cp_compute_enable(struct amdgpu_device *adev, bool enable)
3557 {
3558         u32 data;
3559
3560         if (adev->gfx.rs64_enable) {
3561                 data = RREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL);
3562                 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_INVALIDATE_ICACHE,
3563                                                          enable ? 0 : 1);
3564                 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE0_RESET,
3565                                                          enable ? 0 : 1);
3566                 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE1_RESET,
3567                                                          enable ? 0 : 1);
3568                 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE2_RESET,
3569                                                          enable ? 0 : 1);
3570                 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE3_RESET,
3571                                                          enable ? 0 : 1);
3572                 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE0_ACTIVE,
3573                                                          enable ? 1 : 0);
3574                 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE1_ACTIVE,
3575                                                          enable ? 1 : 0);
3576                 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE2_ACTIVE,
3577                                                          enable ? 1 : 0);
3578                 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE3_ACTIVE,
3579                                                          enable ? 1 : 0);
3580                 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_HALT,
3581                                                          enable ? 0 : 1);
3582                 WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, data);
3583         } else {
3584                 data = RREG32_SOC15(GC, 0, regCP_MEC_CNTL);
3585
3586                 if (enable) {
3587                         data = REG_SET_FIELD(data, CP_MEC_CNTL, MEC_ME1_HALT, 0);
3588                         if (!adev->enable_mes_kiq)
3589                                 data = REG_SET_FIELD(data, CP_MEC_CNTL,
3590                                                      MEC_ME2_HALT, 0);
3591                 } else {
3592                         data = REG_SET_FIELD(data, CP_MEC_CNTL, MEC_ME1_HALT, 1);
3593                         data = REG_SET_FIELD(data, CP_MEC_CNTL, MEC_ME2_HALT, 1);
3594                 }
3595                 WREG32_SOC15(GC, 0, regCP_MEC_CNTL, data);
3596         }
3597
3598         udelay(50);
3599 }
3600
3601 static int gfx_v11_0_cp_compute_load_microcode(struct amdgpu_device *adev)
3602 {
3603         const struct gfx_firmware_header_v1_0 *mec_hdr;
3604         const __le32 *fw_data;
3605         unsigned i, fw_size;
3606         u32 *fw = NULL;
3607         int r;
3608
3609         if (!adev->gfx.mec_fw)
3610                 return -EINVAL;
3611
3612         gfx_v11_0_cp_compute_enable(adev, false);
3613
3614         mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
3615         amdgpu_ucode_print_gfx_hdr(&mec_hdr->header);
3616
3617         fw_data = (const __le32 *)
3618                 (adev->gfx.mec_fw->data +
3619                  le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
3620         fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes);
3621
3622         r = amdgpu_bo_create_reserved(adev, mec_hdr->header.ucode_size_bytes,
3623                                           PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
3624                                           &adev->gfx.mec.mec_fw_obj,
3625                                           &adev->gfx.mec.mec_fw_gpu_addr,
3626                                           (void **)&fw);
3627         if (r) {
3628                 dev_err(adev->dev, "(%d) failed to create mec fw bo\n", r);
3629                 gfx_v11_0_mec_fini(adev);
3630                 return r;
3631         }
3632
3633         memcpy(fw, fw_data, fw_size);
3634
3635         amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj);
3636         amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj);
3637
3638         gfx_v11_0_config_mec_cache(adev, adev->gfx.mec.mec_fw_gpu_addr);
3639
3640         /* MEC1 */
3641         WREG32_SOC15(GC, 0, regCP_MEC_ME1_UCODE_ADDR, 0);
3642
3643         for (i = 0; i < mec_hdr->jt_size; i++)
3644                 WREG32_SOC15(GC, 0, regCP_MEC_ME1_UCODE_DATA,
3645                              le32_to_cpup(fw_data + mec_hdr->jt_offset + i));
3646
3647         WREG32_SOC15(GC, 0, regCP_MEC_ME1_UCODE_ADDR, adev->gfx.mec_fw_version);
3648
3649         return 0;
3650 }
3651
3652 static int gfx_v11_0_cp_compute_load_microcode_rs64(struct amdgpu_device *adev)
3653 {
3654         const struct gfx_firmware_header_v2_0 *mec_hdr;
3655         const __le32 *fw_ucode, *fw_data;
3656         u32 tmp, fw_ucode_size, fw_data_size;
3657         u32 i, usec_timeout = 50000; /* Wait for 50 ms */
3658         u32 *fw_ucode_ptr, *fw_data_ptr;
3659         int r;
3660
3661         if (!adev->gfx.mec_fw)
3662                 return -EINVAL;
3663
3664         gfx_v11_0_cp_compute_enable(adev, false);
3665
3666         mec_hdr = (const struct gfx_firmware_header_v2_0 *)adev->gfx.mec_fw->data;
3667         amdgpu_ucode_print_gfx_hdr(&mec_hdr->header);
3668
3669         fw_ucode = (const __le32 *) (adev->gfx.mec_fw->data +
3670                                 le32_to_cpu(mec_hdr->ucode_offset_bytes));
3671         fw_ucode_size = le32_to_cpu(mec_hdr->ucode_size_bytes);
3672
3673         fw_data = (const __le32 *) (adev->gfx.mec_fw->data +
3674                                 le32_to_cpu(mec_hdr->data_offset_bytes));
3675         fw_data_size = le32_to_cpu(mec_hdr->data_size_bytes);
3676
3677         r = amdgpu_bo_create_reserved(adev, fw_ucode_size,
3678                                       64 * 1024,
3679                                       AMDGPU_GEM_DOMAIN_VRAM |
3680                                       AMDGPU_GEM_DOMAIN_GTT,
3681                                       &adev->gfx.mec.mec_fw_obj,
3682                                       &adev->gfx.mec.mec_fw_gpu_addr,
3683                                       (void **)&fw_ucode_ptr);
3684         if (r) {
3685                 dev_err(adev->dev, "(%d) failed to create mec fw ucode bo\n", r);
3686                 gfx_v11_0_mec_fini(adev);
3687                 return r;
3688         }
3689
3690         r = amdgpu_bo_create_reserved(adev, fw_data_size,
3691                                       64 * 1024,
3692                                       AMDGPU_GEM_DOMAIN_VRAM |
3693                                       AMDGPU_GEM_DOMAIN_GTT,
3694                                       &adev->gfx.mec.mec_fw_data_obj,
3695                                       &adev->gfx.mec.mec_fw_data_gpu_addr,
3696                                       (void **)&fw_data_ptr);
3697         if (r) {
3698                 dev_err(adev->dev, "(%d) failed to create mec fw ucode bo\n", r);
3699                 gfx_v11_0_mec_fini(adev);
3700                 return r;
3701         }
3702
3703         memcpy(fw_ucode_ptr, fw_ucode, fw_ucode_size);
3704         memcpy(fw_data_ptr, fw_data, fw_data_size);
3705
3706         amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj);
3707         amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_data_obj);
3708         amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj);
3709         amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_data_obj);
3710
3711         tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL);
3712         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0);
3713         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, EXE_DISABLE, 0);
3714         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0);
3715         WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL, tmp);
3716
3717         tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_BASE_CNTL);
3718         tmp = REG_SET_FIELD(tmp, CP_MEC_DC_BASE_CNTL, VMID, 0);
3719         tmp = REG_SET_FIELD(tmp, CP_MEC_DC_BASE_CNTL, CACHE_POLICY, 0);
3720         WREG32_SOC15(GC, 0, regCP_MEC_DC_BASE_CNTL, tmp);
3721
3722         mutex_lock(&adev->srbm_mutex);
3723         for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) {
3724                 soc21_grbm_select(adev, 1, i, 0, 0);
3725
3726                 WREG32_SOC15(GC, 0, regCP_MEC_MDBASE_LO, adev->gfx.mec.mec_fw_data_gpu_addr);
3727                 WREG32_SOC15(GC, 0, regCP_MEC_MDBASE_HI,
3728                      upper_32_bits(adev->gfx.mec.mec_fw_data_gpu_addr));
3729
3730                 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START,
3731                                         mec_hdr->ucode_start_addr_lo >> 2 |
3732                                         mec_hdr->ucode_start_addr_hi << 30);
3733                 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START_HI,
3734                                         mec_hdr->ucode_start_addr_hi >> 2);
3735
3736                 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_LO, adev->gfx.mec.mec_fw_gpu_addr);
3737                 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_HI,
3738                      upper_32_bits(adev->gfx.mec.mec_fw_gpu_addr));
3739         }
3740         mutex_unlock(&adev->srbm_mutex);
3741         soc21_grbm_select(adev, 0, 0, 0, 0);
3742
3743         /* Trigger an invalidation of the L1 instruction caches */
3744         tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL);
3745         tmp = REG_SET_FIELD(tmp, CP_MEC_DC_OP_CNTL, INVALIDATE_DCACHE, 1);
3746         WREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL, tmp);
3747
3748         /* Wait for invalidation complete */
3749         for (i = 0; i < usec_timeout; i++) {
3750                 tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL);
3751                 if (1 == REG_GET_FIELD(tmp, CP_MEC_DC_OP_CNTL,
3752                                        INVALIDATE_DCACHE_COMPLETE))
3753                         break;
3754                 udelay(1);
3755         }
3756
3757         if (i >= usec_timeout) {
3758                 dev_err(adev->dev, "failed to invalidate instruction cache\n");
3759                 return -EINVAL;
3760         }
3761
3762         /* Trigger an invalidation of the L1 instruction caches */
3763         tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL);
3764         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_OP_CNTL, INVALIDATE_CACHE, 1);
3765         WREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL, tmp);
3766
3767         /* Wait for invalidation complete */
3768         for (i = 0; i < usec_timeout; i++) {
3769                 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL);
3770                 if (1 == REG_GET_FIELD(tmp, CP_CPC_IC_OP_CNTL,
3771                                        INVALIDATE_CACHE_COMPLETE))
3772                         break;
3773                 udelay(1);
3774         }
3775
3776         if (i >= usec_timeout) {
3777                 dev_err(adev->dev, "failed to invalidate instruction cache\n");
3778                 return -EINVAL;
3779         }
3780
3781         return 0;
3782 }
3783
3784 static void gfx_v11_0_kiq_setting(struct amdgpu_ring *ring)
3785 {
3786         uint32_t tmp;
3787         struct amdgpu_device *adev = ring->adev;
3788
3789         /* tell RLC which is KIQ queue */
3790         tmp = RREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS);
3791         tmp &= 0xffffff00;
3792         tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue);
3793         WREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS, tmp);
3794         tmp |= 0x80;
3795         WREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS, tmp);
3796 }
3797
3798 static void gfx_v11_0_cp_set_doorbell_range(struct amdgpu_device *adev)
3799 {
3800         /* set graphics engine doorbell range */
3801         WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_RANGE_LOWER,
3802                      (adev->doorbell_index.gfx_ring0 * 2) << 2);
3803         WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_RANGE_UPPER,
3804                      (adev->doorbell_index.gfx_userqueue_end * 2) << 2);
3805
3806         /* set compute engine doorbell range */
3807         WREG32_SOC15(GC, 0, regCP_MEC_DOORBELL_RANGE_LOWER,
3808                      (adev->doorbell_index.kiq * 2) << 2);
3809         WREG32_SOC15(GC, 0, regCP_MEC_DOORBELL_RANGE_UPPER,
3810                      (adev->doorbell_index.userqueue_end * 2) << 2);
3811 }
3812
3813 static void gfx_v11_0_gfx_mqd_set_priority(struct amdgpu_device *adev,
3814                                            struct v11_gfx_mqd *mqd,
3815                                            struct amdgpu_mqd_prop *prop)
3816 {
3817         bool priority = 0;
3818         u32 tmp;
3819
3820         /* set up default queue priority level
3821          * 0x0 = low priority, 0x1 = high priority
3822          */
3823         if (prop->hqd_pipe_priority == AMDGPU_GFX_PIPE_PRIO_HIGH)
3824                 priority = 1;
3825
3826         tmp = RREG32_SOC15(GC, 0, regCP_GFX_HQD_QUEUE_PRIORITY);
3827         tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_QUEUE_PRIORITY, PRIORITY_LEVEL, priority);
3828         mqd->cp_gfx_hqd_queue_priority = tmp;
3829 }
3830
3831 static int gfx_v11_0_gfx_mqd_init(struct amdgpu_device *adev, void *m,
3832                                   struct amdgpu_mqd_prop *prop)
3833 {
3834         struct v11_gfx_mqd *mqd = m;
3835         uint64_t hqd_gpu_addr, wb_gpu_addr;
3836         uint32_t tmp;
3837         uint32_t rb_bufsz;
3838
3839         /* set up gfx hqd wptr */
3840         mqd->cp_gfx_hqd_wptr = 0;
3841         mqd->cp_gfx_hqd_wptr_hi = 0;
3842
3843         /* set the pointer to the MQD */
3844         mqd->cp_mqd_base_addr = prop->mqd_gpu_addr & 0xfffffffc;
3845         mqd->cp_mqd_base_addr_hi = upper_32_bits(prop->mqd_gpu_addr);
3846
3847         /* set up mqd control */
3848         tmp = RREG32_SOC15(GC, 0, regCP_GFX_MQD_CONTROL);
3849         tmp = REG_SET_FIELD(tmp, CP_GFX_MQD_CONTROL, VMID, 0);
3850         tmp = REG_SET_FIELD(tmp, CP_GFX_MQD_CONTROL, PRIV_STATE, 1);
3851         tmp = REG_SET_FIELD(tmp, CP_GFX_MQD_CONTROL, CACHE_POLICY, 0);
3852         mqd->cp_gfx_mqd_control = tmp;
3853
3854         /* set up gfx_hqd_vimd with 0x0 to indicate the ring buffer's vmid */
3855         tmp = RREG32_SOC15(GC, 0, regCP_GFX_HQD_VMID);
3856         tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_VMID, VMID, 0);
3857         mqd->cp_gfx_hqd_vmid = 0;
3858
3859         /* set up gfx queue priority */
3860         gfx_v11_0_gfx_mqd_set_priority(adev, mqd, prop);
3861
3862         /* set up time quantum */
3863         tmp = RREG32_SOC15(GC, 0, regCP_GFX_HQD_QUANTUM);
3864         tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_QUANTUM, QUANTUM_EN, 1);
3865         mqd->cp_gfx_hqd_quantum = tmp;
3866
3867         /* set up gfx hqd base. this is similar as CP_RB_BASE */
3868         hqd_gpu_addr = prop->hqd_base_gpu_addr >> 8;
3869         mqd->cp_gfx_hqd_base = hqd_gpu_addr;
3870         mqd->cp_gfx_hqd_base_hi = upper_32_bits(hqd_gpu_addr);
3871
3872         /* set up hqd_rptr_addr/_hi, similar as CP_RB_RPTR */
3873         wb_gpu_addr = prop->rptr_gpu_addr;
3874         mqd->cp_gfx_hqd_rptr_addr = wb_gpu_addr & 0xfffffffc;
3875         mqd->cp_gfx_hqd_rptr_addr_hi =
3876                 upper_32_bits(wb_gpu_addr) & 0xffff;
3877
3878         /* set up rb_wptr_poll addr */
3879         wb_gpu_addr = prop->wptr_gpu_addr;
3880         mqd->cp_rb_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc;
3881         mqd->cp_rb_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
3882
3883         /* set up the gfx_hqd_control, similar as CP_RB0_CNTL */
3884         rb_bufsz = order_base_2(prop->queue_size / 4) - 1;
3885         tmp = RREG32_SOC15(GC, 0, regCP_GFX_HQD_CNTL);
3886         tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, RB_BUFSZ, rb_bufsz);
3887         tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, RB_BLKSZ, rb_bufsz - 2);
3888 #ifdef __BIG_ENDIAN
3889         tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, BUF_SWAP, 1);
3890 #endif
3891         mqd->cp_gfx_hqd_cntl = tmp;
3892
3893         /* set up cp_doorbell_control */
3894         tmp = RREG32_SOC15(GC, 0, regCP_RB_DOORBELL_CONTROL);
3895         if (prop->use_doorbell) {
3896                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3897                                     DOORBELL_OFFSET, prop->doorbell_index);
3898                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3899                                     DOORBELL_EN, 1);
3900         } else
3901                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3902                                     DOORBELL_EN, 0);
3903         mqd->cp_rb_doorbell_control = tmp;
3904
3905         /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
3906         mqd->cp_gfx_hqd_rptr = RREG32_SOC15(GC, 0, regCP_GFX_HQD_RPTR);
3907
3908         /* active the queue */
3909         mqd->cp_gfx_hqd_active = 1;
3910
3911         return 0;
3912 }
3913
3914 static int gfx_v11_0_gfx_init_queue(struct amdgpu_ring *ring)
3915 {
3916         struct amdgpu_device *adev = ring->adev;
3917         struct v11_gfx_mqd *mqd = ring->mqd_ptr;
3918         int mqd_idx = ring - &adev->gfx.gfx_ring[0];
3919
3920         if (!amdgpu_in_reset(adev) && !adev->in_suspend) {
3921                 memset((void *)mqd, 0, sizeof(*mqd));
3922                 mutex_lock(&adev->srbm_mutex);
3923                 soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
3924                 amdgpu_ring_init_mqd(ring);
3925                 soc21_grbm_select(adev, 0, 0, 0, 0);
3926                 mutex_unlock(&adev->srbm_mutex);
3927                 if (adev->gfx.me.mqd_backup[mqd_idx])
3928                         memcpy_fromio(adev->gfx.me.mqd_backup[mqd_idx], mqd, sizeof(*mqd));
3929         } else {
3930                 /* restore mqd with the backup copy */
3931                 if (adev->gfx.me.mqd_backup[mqd_idx])
3932                         memcpy_toio(mqd, adev->gfx.me.mqd_backup[mqd_idx], sizeof(*mqd));
3933                 /* reset the ring */
3934                 ring->wptr = 0;
3935                 *ring->wptr_cpu_addr = 0;
3936                 amdgpu_ring_clear_ring(ring);
3937         }
3938
3939         return 0;
3940 }
3941
3942 static int gfx_v11_0_cp_async_gfx_ring_resume(struct amdgpu_device *adev)
3943 {
3944         int r, i;
3945         struct amdgpu_ring *ring;
3946
3947         for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
3948                 ring = &adev->gfx.gfx_ring[i];
3949
3950                 r = amdgpu_bo_reserve(ring->mqd_obj, false);
3951                 if (unlikely(r != 0))
3952                         return r;
3953
3954                 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
3955                 if (!r) {
3956                         r = gfx_v11_0_gfx_init_queue(ring);
3957                         amdgpu_bo_kunmap(ring->mqd_obj);
3958                         ring->mqd_ptr = NULL;
3959                 }
3960                 amdgpu_bo_unreserve(ring->mqd_obj);
3961                 if (r)
3962                         return r;
3963         }
3964
3965         r = amdgpu_gfx_enable_kgq(adev, 0);
3966         if (r)
3967                 return r;
3968
3969         return gfx_v11_0_cp_gfx_start(adev);
3970 }
3971
3972 static int gfx_v11_0_compute_mqd_init(struct amdgpu_device *adev, void *m,
3973                                       struct amdgpu_mqd_prop *prop)
3974 {
3975         struct v11_compute_mqd *mqd = m;
3976         uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr;
3977         uint32_t tmp;
3978
3979         mqd->header = 0xC0310800;
3980         mqd->compute_pipelinestat_enable = 0x00000001;
3981         mqd->compute_static_thread_mgmt_se0 = 0xffffffff;
3982         mqd->compute_static_thread_mgmt_se1 = 0xffffffff;
3983         mqd->compute_static_thread_mgmt_se2 = 0xffffffff;
3984         mqd->compute_static_thread_mgmt_se3 = 0xffffffff;
3985         mqd->compute_misc_reserved = 0x00000007;
3986
3987         eop_base_addr = prop->eop_gpu_addr >> 8;
3988         mqd->cp_hqd_eop_base_addr_lo = eop_base_addr;
3989         mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr);
3990
3991         /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
3992         tmp = RREG32_SOC15(GC, 0, regCP_HQD_EOP_CONTROL);
3993         tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE,
3994                         (order_base_2(GFX11_MEC_HPD_SIZE / 4) - 1));
3995
3996         mqd->cp_hqd_eop_control = tmp;
3997
3998         /* enable doorbell? */
3999         tmp = RREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL);
4000
4001         if (prop->use_doorbell) {
4002                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4003                                     DOORBELL_OFFSET, prop->doorbell_index);
4004                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4005                                     DOORBELL_EN, 1);
4006                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4007                                     DOORBELL_SOURCE, 0);
4008                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4009                                     DOORBELL_HIT, 0);
4010         } else {
4011                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4012                                     DOORBELL_EN, 0);
4013         }
4014
4015         mqd->cp_hqd_pq_doorbell_control = tmp;
4016
4017         /* disable the queue if it's active */
4018         mqd->cp_hqd_dequeue_request = 0;
4019         mqd->cp_hqd_pq_rptr = 0;
4020         mqd->cp_hqd_pq_wptr_lo = 0;
4021         mqd->cp_hqd_pq_wptr_hi = 0;
4022
4023         /* set the pointer to the MQD */
4024         mqd->cp_mqd_base_addr_lo = prop->mqd_gpu_addr & 0xfffffffc;
4025         mqd->cp_mqd_base_addr_hi = upper_32_bits(prop->mqd_gpu_addr);
4026
4027         /* set MQD vmid to 0 */
4028         tmp = RREG32_SOC15(GC, 0, regCP_MQD_CONTROL);
4029         tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0);
4030         mqd->cp_mqd_control = tmp;
4031
4032         /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
4033         hqd_gpu_addr = prop->hqd_base_gpu_addr >> 8;
4034         mqd->cp_hqd_pq_base_lo = hqd_gpu_addr;
4035         mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
4036
4037         /* set up the HQD, this is similar to CP_RB0_CNTL */
4038         tmp = RREG32_SOC15(GC, 0, regCP_HQD_PQ_CONTROL);
4039         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE,
4040                             (order_base_2(prop->queue_size / 4) - 1));
4041         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE,
4042                             (order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1));
4043         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 1);
4044         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, TUNNEL_DISPATCH,
4045                             prop->allow_tunneling);
4046         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1);
4047         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1);
4048         mqd->cp_hqd_pq_control = tmp;
4049
4050         /* set the wb address whether it's enabled or not */
4051         wb_gpu_addr = prop->rptr_gpu_addr;
4052         mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc;
4053         mqd->cp_hqd_pq_rptr_report_addr_hi =
4054                 upper_32_bits(wb_gpu_addr) & 0xffff;
4055
4056         /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
4057         wb_gpu_addr = prop->wptr_gpu_addr;
4058         mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc;
4059         mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
4060
4061         tmp = 0;
4062         /* enable the doorbell if requested */
4063         if (prop->use_doorbell) {
4064                 tmp = RREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL);
4065                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4066                                 DOORBELL_OFFSET, prop->doorbell_index);
4067
4068                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4069                                     DOORBELL_EN, 1);
4070                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4071                                     DOORBELL_SOURCE, 0);
4072                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4073                                     DOORBELL_HIT, 0);
4074         }
4075
4076         mqd->cp_hqd_pq_doorbell_control = tmp;
4077
4078         /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
4079         mqd->cp_hqd_pq_rptr = RREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR);
4080
4081         /* set the vmid for the queue */
4082         mqd->cp_hqd_vmid = 0;
4083
4084         tmp = RREG32_SOC15(GC, 0, regCP_HQD_PERSISTENT_STATE);
4085         tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x55);
4086         mqd->cp_hqd_persistent_state = tmp;
4087
4088         /* set MIN_IB_AVAIL_SIZE */
4089         tmp = RREG32_SOC15(GC, 0, regCP_HQD_IB_CONTROL);
4090         tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 3);
4091         mqd->cp_hqd_ib_control = tmp;
4092
4093         /* set static priority for a compute queue/ring */
4094         mqd->cp_hqd_pipe_priority = prop->hqd_pipe_priority;
4095         mqd->cp_hqd_queue_priority = prop->hqd_queue_priority;
4096
4097         mqd->cp_hqd_active = prop->hqd_active;
4098
4099         return 0;
4100 }
4101
4102 static int gfx_v11_0_kiq_init_register(struct amdgpu_ring *ring)
4103 {
4104         struct amdgpu_device *adev = ring->adev;
4105         struct v11_compute_mqd *mqd = ring->mqd_ptr;
4106         int j;
4107
4108         /* inactivate the queue */
4109         if (amdgpu_sriov_vf(adev))
4110                 WREG32_SOC15(GC, 0, regCP_HQD_ACTIVE, 0);
4111
4112         /* disable wptr polling */
4113         WREG32_FIELD15_PREREG(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0);
4114
4115         /* write the EOP addr */
4116         WREG32_SOC15(GC, 0, regCP_HQD_EOP_BASE_ADDR,
4117                mqd->cp_hqd_eop_base_addr_lo);
4118         WREG32_SOC15(GC, 0, regCP_HQD_EOP_BASE_ADDR_HI,
4119                mqd->cp_hqd_eop_base_addr_hi);
4120
4121         /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4122         WREG32_SOC15(GC, 0, regCP_HQD_EOP_CONTROL,
4123                mqd->cp_hqd_eop_control);
4124
4125         /* enable doorbell? */
4126         WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL,
4127                mqd->cp_hqd_pq_doorbell_control);
4128
4129         /* disable the queue if it's active */
4130         if (RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) & 1) {
4131                 WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST, 1);
4132                 for (j = 0; j < adev->usec_timeout; j++) {
4133                         if (!(RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) & 1))
4134                                 break;
4135                         udelay(1);
4136                 }
4137                 WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST,
4138                        mqd->cp_hqd_dequeue_request);
4139                 WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR,
4140                        mqd->cp_hqd_pq_rptr);
4141                 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_LO,
4142                        mqd->cp_hqd_pq_wptr_lo);
4143                 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_HI,
4144                        mqd->cp_hqd_pq_wptr_hi);
4145         }
4146
4147         /* set the pointer to the MQD */
4148         WREG32_SOC15(GC, 0, regCP_MQD_BASE_ADDR,
4149                mqd->cp_mqd_base_addr_lo);
4150         WREG32_SOC15(GC, 0, regCP_MQD_BASE_ADDR_HI,
4151                mqd->cp_mqd_base_addr_hi);
4152
4153         /* set MQD vmid to 0 */
4154         WREG32_SOC15(GC, 0, regCP_MQD_CONTROL,
4155                mqd->cp_mqd_control);
4156
4157         /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
4158         WREG32_SOC15(GC, 0, regCP_HQD_PQ_BASE,
4159                mqd->cp_hqd_pq_base_lo);
4160         WREG32_SOC15(GC, 0, regCP_HQD_PQ_BASE_HI,
4161                mqd->cp_hqd_pq_base_hi);
4162
4163         /* set up the HQD, this is similar to CP_RB0_CNTL */
4164         WREG32_SOC15(GC, 0, regCP_HQD_PQ_CONTROL,
4165                mqd->cp_hqd_pq_control);
4166
4167         /* set the wb address whether it's enabled or not */
4168         WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR_REPORT_ADDR,
4169                 mqd->cp_hqd_pq_rptr_report_addr_lo);
4170         WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR_REPORT_ADDR_HI,
4171                 mqd->cp_hqd_pq_rptr_report_addr_hi);
4172
4173         /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
4174         WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR,
4175                mqd->cp_hqd_pq_wptr_poll_addr_lo);
4176         WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR_HI,
4177                mqd->cp_hqd_pq_wptr_poll_addr_hi);
4178
4179         /* enable the doorbell if requested */
4180         if (ring->use_doorbell) {
4181                 WREG32_SOC15(GC, 0, regCP_MEC_DOORBELL_RANGE_LOWER,
4182                         (adev->doorbell_index.kiq * 2) << 2);
4183                 WREG32_SOC15(GC, 0, regCP_MEC_DOORBELL_RANGE_UPPER,
4184                         (adev->doorbell_index.userqueue_end * 2) << 2);
4185         }
4186
4187         WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL,
4188                mqd->cp_hqd_pq_doorbell_control);
4189
4190         /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
4191         WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_LO,
4192                mqd->cp_hqd_pq_wptr_lo);
4193         WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_HI,
4194                mqd->cp_hqd_pq_wptr_hi);
4195
4196         /* set the vmid for the queue */
4197         WREG32_SOC15(GC, 0, regCP_HQD_VMID, mqd->cp_hqd_vmid);
4198
4199         WREG32_SOC15(GC, 0, regCP_HQD_PERSISTENT_STATE,
4200                mqd->cp_hqd_persistent_state);
4201
4202         /* activate the queue */
4203         WREG32_SOC15(GC, 0, regCP_HQD_ACTIVE,
4204                mqd->cp_hqd_active);
4205
4206         if (ring->use_doorbell)
4207                 WREG32_FIELD15_PREREG(GC, 0, CP_PQ_STATUS, DOORBELL_ENABLE, 1);
4208
4209         return 0;
4210 }
4211
4212 static int gfx_v11_0_kiq_init_queue(struct amdgpu_ring *ring)
4213 {
4214         struct amdgpu_device *adev = ring->adev;
4215         struct v11_compute_mqd *mqd = ring->mqd_ptr;
4216
4217         gfx_v11_0_kiq_setting(ring);
4218
4219         if (amdgpu_in_reset(adev)) { /* for GPU_RESET case */
4220                 /* reset MQD to a clean status */
4221                 if (adev->gfx.kiq[0].mqd_backup)
4222                         memcpy_toio(mqd, adev->gfx.kiq[0].mqd_backup, sizeof(*mqd));
4223
4224                 /* reset ring buffer */
4225                 ring->wptr = 0;
4226                 amdgpu_ring_clear_ring(ring);
4227
4228                 mutex_lock(&adev->srbm_mutex);
4229                 soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
4230                 gfx_v11_0_kiq_init_register(ring);
4231                 soc21_grbm_select(adev, 0, 0, 0, 0);
4232                 mutex_unlock(&adev->srbm_mutex);
4233         } else {
4234                 memset((void *)mqd, 0, sizeof(*mqd));
4235                 if (amdgpu_sriov_vf(adev) && adev->in_suspend)
4236                         amdgpu_ring_clear_ring(ring);
4237                 mutex_lock(&adev->srbm_mutex);
4238                 soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
4239                 amdgpu_ring_init_mqd(ring);
4240                 gfx_v11_0_kiq_init_register(ring);
4241                 soc21_grbm_select(adev, 0, 0, 0, 0);
4242                 mutex_unlock(&adev->srbm_mutex);
4243
4244                 if (adev->gfx.kiq[0].mqd_backup)
4245                         memcpy_fromio(adev->gfx.kiq[0].mqd_backup, mqd, sizeof(*mqd));
4246         }
4247
4248         return 0;
4249 }
4250
4251 static int gfx_v11_0_kcq_init_queue(struct amdgpu_ring *ring)
4252 {
4253         struct amdgpu_device *adev = ring->adev;
4254         struct v11_compute_mqd *mqd = ring->mqd_ptr;
4255         int mqd_idx = ring - &adev->gfx.compute_ring[0];
4256
4257         if (!amdgpu_in_reset(adev) && !adev->in_suspend) {
4258                 memset((void *)mqd, 0, sizeof(*mqd));
4259                 mutex_lock(&adev->srbm_mutex);
4260                 soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
4261                 amdgpu_ring_init_mqd(ring);
4262                 soc21_grbm_select(adev, 0, 0, 0, 0);
4263                 mutex_unlock(&adev->srbm_mutex);
4264
4265                 if (adev->gfx.mec.mqd_backup[mqd_idx])
4266                         memcpy_fromio(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(*mqd));
4267         } else {
4268                 /* restore MQD to a clean status */
4269                 if (adev->gfx.mec.mqd_backup[mqd_idx])
4270                         memcpy_toio(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(*mqd));
4271                 /* reset ring buffer */
4272                 ring->wptr = 0;
4273                 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, 0);
4274                 amdgpu_ring_clear_ring(ring);
4275         }
4276
4277         return 0;
4278 }
4279
4280 static int gfx_v11_0_kiq_resume(struct amdgpu_device *adev)
4281 {
4282         struct amdgpu_ring *ring;
4283         int r;
4284
4285         ring = &adev->gfx.kiq[0].ring;
4286
4287         r = amdgpu_bo_reserve(ring->mqd_obj, false);
4288         if (unlikely(r != 0))
4289                 return r;
4290
4291         r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
4292         if (unlikely(r != 0)) {
4293                 amdgpu_bo_unreserve(ring->mqd_obj);
4294                 return r;
4295         }
4296
4297         gfx_v11_0_kiq_init_queue(ring);
4298         amdgpu_bo_kunmap(ring->mqd_obj);
4299         ring->mqd_ptr = NULL;
4300         amdgpu_bo_unreserve(ring->mqd_obj);
4301         ring->sched.ready = true;
4302         return 0;
4303 }
4304
4305 static int gfx_v11_0_kcq_resume(struct amdgpu_device *adev)
4306 {
4307         struct amdgpu_ring *ring = NULL;
4308         int r = 0, i;
4309
4310         if (!amdgpu_async_gfx_ring)
4311                 gfx_v11_0_cp_compute_enable(adev, true);
4312
4313         for (i = 0; i < adev->gfx.num_compute_rings; i++) {
4314                 ring = &adev->gfx.compute_ring[i];
4315
4316                 r = amdgpu_bo_reserve(ring->mqd_obj, false);
4317                 if (unlikely(r != 0))
4318                         goto done;
4319                 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
4320                 if (!r) {
4321                         r = gfx_v11_0_kcq_init_queue(ring);
4322                         amdgpu_bo_kunmap(ring->mqd_obj);
4323                         ring->mqd_ptr = NULL;
4324                 }
4325                 amdgpu_bo_unreserve(ring->mqd_obj);
4326                 if (r)
4327                         goto done;
4328         }
4329
4330         r = amdgpu_gfx_enable_kcq(adev, 0);
4331 done:
4332         return r;
4333 }
4334
4335 static int gfx_v11_0_cp_resume(struct amdgpu_device *adev)
4336 {
4337         int r, i;
4338         struct amdgpu_ring *ring;
4339
4340         if (!(adev->flags & AMD_IS_APU))
4341                 gfx_v11_0_enable_gui_idle_interrupt(adev, false);
4342
4343         if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
4344                 /* legacy firmware loading */
4345                 r = gfx_v11_0_cp_gfx_load_microcode(adev);
4346                 if (r)
4347                         return r;
4348
4349                 if (adev->gfx.rs64_enable)
4350                         r = gfx_v11_0_cp_compute_load_microcode_rs64(adev);
4351                 else
4352                         r = gfx_v11_0_cp_compute_load_microcode(adev);
4353                 if (r)
4354                         return r;
4355         }
4356
4357         gfx_v11_0_cp_set_doorbell_range(adev);
4358
4359         if (amdgpu_async_gfx_ring) {
4360                 gfx_v11_0_cp_compute_enable(adev, true);
4361                 gfx_v11_0_cp_gfx_enable(adev, true);
4362         }
4363
4364         if (adev->enable_mes_kiq && adev->mes.kiq_hw_init)
4365                 r = amdgpu_mes_kiq_hw_init(adev);
4366         else
4367                 r = gfx_v11_0_kiq_resume(adev);
4368         if (r)
4369                 return r;
4370
4371         r = gfx_v11_0_kcq_resume(adev);
4372         if (r)
4373                 return r;
4374
4375         if (!amdgpu_async_gfx_ring) {
4376                 r = gfx_v11_0_cp_gfx_resume(adev);
4377                 if (r)
4378                         return r;
4379         } else {
4380                 r = gfx_v11_0_cp_async_gfx_ring_resume(adev);
4381                 if (r)
4382                         return r;
4383         }
4384
4385         for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
4386                 ring = &adev->gfx.gfx_ring[i];
4387                 r = amdgpu_ring_test_helper(ring);
4388                 if (r)
4389                         return r;
4390         }
4391
4392         for (i = 0; i < adev->gfx.num_compute_rings; i++) {
4393                 ring = &adev->gfx.compute_ring[i];
4394                 r = amdgpu_ring_test_helper(ring);
4395                 if (r)
4396                         return r;
4397         }
4398
4399         return 0;
4400 }
4401
4402 static void gfx_v11_0_cp_enable(struct amdgpu_device *adev, bool enable)
4403 {
4404         gfx_v11_0_cp_gfx_enable(adev, enable);
4405         gfx_v11_0_cp_compute_enable(adev, enable);
4406 }
4407
4408 static int gfx_v11_0_gfxhub_enable(struct amdgpu_device *adev)
4409 {
4410         int r;
4411         bool value;
4412
4413         r = adev->gfxhub.funcs->gart_enable(adev);
4414         if (r)
4415                 return r;
4416
4417         adev->hdp.funcs->flush_hdp(adev, NULL);
4418
4419         value = (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS) ?
4420                 false : true;
4421
4422         adev->gfxhub.funcs->set_fault_enable_default(adev, value);
4423         /* TODO investigate why this and the hdp flush above is needed,
4424          * are we missing a flush somewhere else? */
4425         adev->gmc.gmc_funcs->flush_gpu_tlb(adev, 0, AMDGPU_GFXHUB(0), 0);
4426
4427         return 0;
4428 }
4429
4430 static void gfx_v11_0_select_cp_fw_arch(struct amdgpu_device *adev)
4431 {
4432         u32 tmp;
4433
4434         /* select RS64 */
4435         if (adev->gfx.rs64_enable) {
4436                 tmp = RREG32_SOC15(GC, 0, regCP_GFX_CNTL);
4437                 tmp = REG_SET_FIELD(tmp, CP_GFX_CNTL, ENGINE_SEL, 1);
4438                 WREG32_SOC15(GC, 0, regCP_GFX_CNTL, tmp);
4439
4440                 tmp = RREG32_SOC15(GC, 0, regCP_MEC_ISA_CNTL);
4441                 tmp = REG_SET_FIELD(tmp, CP_MEC_ISA_CNTL, ISA_MODE, 1);
4442                 WREG32_SOC15(GC, 0, regCP_MEC_ISA_CNTL, tmp);
4443         }
4444
4445         if (amdgpu_emu_mode == 1)
4446                 msleep(100);
4447 }
4448
4449 static int get_gb_addr_config(struct amdgpu_device * adev)
4450 {
4451         u32 gb_addr_config;
4452
4453         gb_addr_config = RREG32_SOC15(GC, 0, regGB_ADDR_CONFIG);
4454         if (gb_addr_config == 0)
4455                 return -EINVAL;
4456
4457         adev->gfx.config.gb_addr_config_fields.num_pkrs =
4458                 1 << REG_GET_FIELD(gb_addr_config, GB_ADDR_CONFIG, NUM_PKRS);
4459
4460         adev->gfx.config.gb_addr_config = gb_addr_config;
4461
4462         adev->gfx.config.gb_addr_config_fields.num_pipes = 1 <<
4463                         REG_GET_FIELD(adev->gfx.config.gb_addr_config,
4464                                       GB_ADDR_CONFIG, NUM_PIPES);
4465
4466         adev->gfx.config.max_tile_pipes =
4467                 adev->gfx.config.gb_addr_config_fields.num_pipes;
4468
4469         adev->gfx.config.gb_addr_config_fields.max_compress_frags = 1 <<
4470                         REG_GET_FIELD(adev->gfx.config.gb_addr_config,
4471                                       GB_ADDR_CONFIG, MAX_COMPRESSED_FRAGS);
4472         adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 1 <<
4473                         REG_GET_FIELD(adev->gfx.config.gb_addr_config,
4474                                       GB_ADDR_CONFIG, NUM_RB_PER_SE);
4475         adev->gfx.config.gb_addr_config_fields.num_se = 1 <<
4476                         REG_GET_FIELD(adev->gfx.config.gb_addr_config,
4477                                       GB_ADDR_CONFIG, NUM_SHADER_ENGINES);
4478         adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 1 << (8 +
4479                         REG_GET_FIELD(adev->gfx.config.gb_addr_config,
4480                                       GB_ADDR_CONFIG, PIPE_INTERLEAVE_SIZE));
4481
4482         return 0;
4483 }
4484
4485 static void gfx_v11_0_disable_gpa_mode(struct amdgpu_device *adev)
4486 {
4487         uint32_t data;
4488
4489         data = RREG32_SOC15(GC, 0, regCPC_PSP_DEBUG);
4490         data |= CPC_PSP_DEBUG__GPA_OVERRIDE_MASK;
4491         WREG32_SOC15(GC, 0, regCPC_PSP_DEBUG, data);
4492
4493         data = RREG32_SOC15(GC, 0, regCPG_PSP_DEBUG);
4494         data |= CPG_PSP_DEBUG__GPA_OVERRIDE_MASK;
4495         WREG32_SOC15(GC, 0, regCPG_PSP_DEBUG, data);
4496 }
4497
4498 static int gfx_v11_0_hw_init(void *handle)
4499 {
4500         int r;
4501         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4502
4503         if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) {
4504                 if (adev->gfx.imu.funcs) {
4505                         /* RLC autoload sequence 1: Program rlc ram */
4506                         if (adev->gfx.imu.funcs->program_rlc_ram)
4507                                 adev->gfx.imu.funcs->program_rlc_ram(adev);
4508                         /* rlc autoload firmware */
4509                         r = gfx_v11_0_rlc_backdoor_autoload_enable(adev);
4510                         if (r)
4511                                 return r;
4512                 }
4513         } else {
4514                 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
4515                         if (adev->gfx.imu.funcs && (amdgpu_dpm > 0)) {
4516                                 if (adev->gfx.imu.funcs->load_microcode)
4517                                         adev->gfx.imu.funcs->load_microcode(adev);
4518                                 if (adev->gfx.imu.funcs->setup_imu)
4519                                         adev->gfx.imu.funcs->setup_imu(adev);
4520                                 if (adev->gfx.imu.funcs->start_imu)
4521                                         adev->gfx.imu.funcs->start_imu(adev);
4522                         }
4523
4524                         /* disable gpa mode in backdoor loading */
4525                         gfx_v11_0_disable_gpa_mode(adev);
4526                 }
4527         }
4528
4529         if ((adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) ||
4530             (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP)) {
4531                 r = gfx_v11_0_wait_for_rlc_autoload_complete(adev);
4532                 if (r) {
4533                         dev_err(adev->dev, "(%d) failed to wait rlc autoload complete\n", r);
4534                         return r;
4535                 }
4536         }
4537
4538         adev->gfx.is_poweron = true;
4539
4540         if(get_gb_addr_config(adev))
4541                 DRM_WARN("Invalid gb_addr_config !\n");
4542
4543         if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP &&
4544             adev->gfx.rs64_enable)
4545                 gfx_v11_0_config_gfx_rs64(adev);
4546
4547         r = gfx_v11_0_gfxhub_enable(adev);
4548         if (r)
4549                 return r;
4550
4551         if (!amdgpu_emu_mode)
4552                 gfx_v11_0_init_golden_registers(adev);
4553
4554         if ((adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) ||
4555             (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO && amdgpu_dpm == 1)) {
4556                 /**
4557                  * For gfx 11, rlc firmware loading relies on smu firmware is
4558                  * loaded firstly, so in direct type, it has to load smc ucode
4559                  * here before rlc.
4560                  */
4561                 r = amdgpu_pm_load_smu_firmware(adev, NULL);
4562                 if (r)
4563                         return r;
4564         }
4565
4566         gfx_v11_0_constants_init(adev);
4567
4568         if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP)
4569                 gfx_v11_0_select_cp_fw_arch(adev);
4570
4571         if (adev->nbio.funcs->gc_doorbell_init)
4572                 adev->nbio.funcs->gc_doorbell_init(adev);
4573
4574         r = gfx_v11_0_rlc_resume(adev);
4575         if (r)
4576                 return r;
4577
4578         /*
4579          * init golden registers and rlc resume may override some registers,
4580          * reconfig them here
4581          */
4582         gfx_v11_0_tcp_harvest(adev);
4583
4584         r = gfx_v11_0_cp_resume(adev);
4585         if (r)
4586                 return r;
4587
4588         /* get IMU version from HW if it's not set */
4589         if (!adev->gfx.imu_fw_version)
4590                 adev->gfx.imu_fw_version = RREG32_SOC15(GC, 0, regGFX_IMU_SCRATCH_0);
4591
4592         return r;
4593 }
4594
4595 static int gfx_v11_0_hw_fini(void *handle)
4596 {
4597         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4598
4599         amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
4600         amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
4601
4602         if (!adev->no_hw_access) {
4603                 if (amdgpu_async_gfx_ring) {
4604                         if (amdgpu_gfx_disable_kgq(adev, 0))
4605                                 DRM_ERROR("KGQ disable failed\n");
4606                 }
4607
4608                 if (amdgpu_gfx_disable_kcq(adev, 0))
4609                         DRM_ERROR("KCQ disable failed\n");
4610
4611                 amdgpu_mes_kiq_hw_fini(adev);
4612         }
4613
4614         if (amdgpu_sriov_vf(adev))
4615                 /* Remove the steps disabling CPG and clearing KIQ position,
4616                  * so that CP could perform IDLE-SAVE during switch. Those
4617                  * steps are necessary to avoid a DMAR error in gfx9 but it is
4618                  * not reproduced on gfx11.
4619                  */
4620                 return 0;
4621
4622         gfx_v11_0_cp_enable(adev, false);
4623         gfx_v11_0_enable_gui_idle_interrupt(adev, false);
4624
4625         adev->gfxhub.funcs->gart_disable(adev);
4626
4627         adev->gfx.is_poweron = false;
4628
4629         return 0;
4630 }
4631
4632 static int gfx_v11_0_suspend(void *handle)
4633 {
4634         return gfx_v11_0_hw_fini(handle);
4635 }
4636
4637 static int gfx_v11_0_resume(void *handle)
4638 {
4639         return gfx_v11_0_hw_init(handle);
4640 }
4641
4642 static bool gfx_v11_0_is_idle(void *handle)
4643 {
4644         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4645
4646         if (REG_GET_FIELD(RREG32_SOC15(GC, 0, regGRBM_STATUS),
4647                                 GRBM_STATUS, GUI_ACTIVE))
4648                 return false;
4649         else
4650                 return true;
4651 }
4652
4653 static int gfx_v11_0_wait_for_idle(void *handle)
4654 {
4655         unsigned i;
4656         u32 tmp;
4657         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4658
4659         for (i = 0; i < adev->usec_timeout; i++) {
4660                 /* read MC_STATUS */
4661                 tmp = RREG32_SOC15(GC, 0, regGRBM_STATUS) &
4662                         GRBM_STATUS__GUI_ACTIVE_MASK;
4663
4664                 if (!REG_GET_FIELD(tmp, GRBM_STATUS, GUI_ACTIVE))
4665                         return 0;
4666                 udelay(1);
4667         }
4668         return -ETIMEDOUT;
4669 }
4670
4671 static int gfx_v11_0_request_gfx_index_mutex(struct amdgpu_device *adev,
4672                                              int req)
4673 {
4674         u32 i, tmp, val;
4675
4676         for (i = 0; i < adev->usec_timeout; i++) {
4677                 /* Request with MeId=2, PipeId=0 */
4678                 tmp = REG_SET_FIELD(0, CP_GFX_INDEX_MUTEX, REQUEST, req);
4679                 tmp = REG_SET_FIELD(tmp, CP_GFX_INDEX_MUTEX, CLIENTID, 4);
4680                 WREG32_SOC15(GC, 0, regCP_GFX_INDEX_MUTEX, tmp);
4681
4682                 val = RREG32_SOC15(GC, 0, regCP_GFX_INDEX_MUTEX);
4683                 if (req) {
4684                         if (val == tmp)
4685                                 break;
4686                 } else {
4687                         tmp = REG_SET_FIELD(tmp, CP_GFX_INDEX_MUTEX,
4688                                             REQUEST, 1);
4689
4690                         /* unlocked or locked by firmware */
4691                         if (val != tmp)
4692                                 break;
4693                 }
4694                 udelay(1);
4695         }
4696
4697         if (i >= adev->usec_timeout)
4698                 return -EINVAL;
4699
4700         return 0;
4701 }
4702
4703 static int gfx_v11_0_soft_reset(void *handle)
4704 {
4705         u32 grbm_soft_reset = 0;
4706         u32 tmp;
4707         int r, i, j, k;
4708         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4709
4710         tmp = RREG32_SOC15(GC, 0, regCP_INT_CNTL);
4711         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CMP_BUSY_INT_ENABLE, 0);
4712         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CNTX_BUSY_INT_ENABLE, 0);
4713         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CNTX_EMPTY_INT_ENABLE, 0);
4714         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, GFX_IDLE_INT_ENABLE, 0);
4715         WREG32_SOC15(GC, 0, regCP_INT_CNTL, tmp);
4716
4717         gfx_v11_0_set_safe_mode(adev, 0);
4718
4719         mutex_lock(&adev->srbm_mutex);
4720         for (i = 0; i < adev->gfx.mec.num_mec; ++i) {
4721                 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {
4722                         for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {
4723                                 soc21_grbm_select(adev, i, k, j, 0);
4724
4725                                 WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST, 0x2);
4726                                 WREG32_SOC15(GC, 0, regSPI_COMPUTE_QUEUE_RESET, 0x1);
4727                         }
4728                 }
4729         }
4730         for (i = 0; i < adev->gfx.me.num_me; ++i) {
4731                 for (j = 0; j < adev->gfx.me.num_queue_per_pipe; j++) {
4732                         for (k = 0; k < adev->gfx.me.num_pipe_per_me; k++) {
4733                                 soc21_grbm_select(adev, i, k, j, 0);
4734
4735                                 WREG32_SOC15(GC, 0, regCP_GFX_HQD_DEQUEUE_REQUEST, 0x1);
4736                         }
4737                 }
4738         }
4739         soc21_grbm_select(adev, 0, 0, 0, 0);
4740         mutex_unlock(&adev->srbm_mutex);
4741
4742         /* Try to acquire the gfx mutex before access to CP_VMID_RESET */
4743         r = gfx_v11_0_request_gfx_index_mutex(adev, 1);
4744         if (r) {
4745                 DRM_ERROR("Failed to acquire the gfx mutex during soft reset\n");
4746                 return r;
4747         }
4748
4749         WREG32_SOC15(GC, 0, regCP_VMID_RESET, 0xfffffffe);
4750
4751         // Read CP_VMID_RESET register three times.
4752         // to get sufficient time for GFX_HQD_ACTIVE reach 0
4753         RREG32_SOC15(GC, 0, regCP_VMID_RESET);
4754         RREG32_SOC15(GC, 0, regCP_VMID_RESET);
4755         RREG32_SOC15(GC, 0, regCP_VMID_RESET);
4756
4757         /* release the gfx mutex */
4758         r = gfx_v11_0_request_gfx_index_mutex(adev, 0);
4759         if (r) {
4760                 DRM_ERROR("Failed to release the gfx mutex during soft reset\n");
4761                 return r;
4762         }
4763
4764         for (i = 0; i < adev->usec_timeout; i++) {
4765                 if (!RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) &&
4766                     !RREG32_SOC15(GC, 0, regCP_GFX_HQD_ACTIVE))
4767                         break;
4768                 udelay(1);
4769         }
4770         if (i >= adev->usec_timeout) {
4771                 printk("Failed to wait all pipes clean\n");
4772                 return -EINVAL;
4773         }
4774
4775         /**********  trigger soft reset  ***********/
4776         grbm_soft_reset = RREG32_SOC15(GC, 0, regGRBM_SOFT_RESET);
4777         grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
4778                                         SOFT_RESET_CP, 1);
4779         grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
4780                                         SOFT_RESET_GFX, 1);
4781         grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
4782                                         SOFT_RESET_CPF, 1);
4783         grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
4784                                         SOFT_RESET_CPC, 1);
4785         grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
4786                                         SOFT_RESET_CPG, 1);
4787         WREG32_SOC15(GC, 0, regGRBM_SOFT_RESET, grbm_soft_reset);
4788         /**********  exit soft reset  ***********/
4789         grbm_soft_reset = RREG32_SOC15(GC, 0, regGRBM_SOFT_RESET);
4790         grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
4791                                         SOFT_RESET_CP, 0);
4792         grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
4793                                         SOFT_RESET_GFX, 0);
4794         grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
4795                                         SOFT_RESET_CPF, 0);
4796         grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
4797                                         SOFT_RESET_CPC, 0);
4798         grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
4799                                         SOFT_RESET_CPG, 0);
4800         WREG32_SOC15(GC, 0, regGRBM_SOFT_RESET, grbm_soft_reset);
4801
4802         tmp = RREG32_SOC15(GC, 0, regCP_SOFT_RESET_CNTL);
4803         tmp = REG_SET_FIELD(tmp, CP_SOFT_RESET_CNTL, CMP_HQD_REG_RESET, 0x1);
4804         WREG32_SOC15(GC, 0, regCP_SOFT_RESET_CNTL, tmp);
4805
4806         WREG32_SOC15(GC, 0, regCP_ME_CNTL, 0x0);
4807         WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, 0x0);
4808
4809         for (i = 0; i < adev->usec_timeout; i++) {
4810                 if (!RREG32_SOC15(GC, 0, regCP_VMID_RESET))
4811                         break;
4812                 udelay(1);
4813         }
4814         if (i >= adev->usec_timeout) {
4815                 printk("Failed to wait CP_VMID_RESET to 0\n");
4816                 return -EINVAL;
4817         }
4818
4819         tmp = RREG32_SOC15(GC, 0, regCP_INT_CNTL);
4820         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CMP_BUSY_INT_ENABLE, 1);
4821         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CNTX_BUSY_INT_ENABLE, 1);
4822         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CNTX_EMPTY_INT_ENABLE, 1);
4823         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, GFX_IDLE_INT_ENABLE, 1);
4824         WREG32_SOC15(GC, 0, regCP_INT_CNTL, tmp);
4825
4826         gfx_v11_0_unset_safe_mode(adev, 0);
4827
4828         return gfx_v11_0_cp_resume(adev);
4829 }
4830
4831 static bool gfx_v11_0_check_soft_reset(void *handle)
4832 {
4833         int i, r;
4834         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4835         struct amdgpu_ring *ring;
4836         long tmo = msecs_to_jiffies(1000);
4837
4838         for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
4839                 ring = &adev->gfx.gfx_ring[i];
4840                 r = amdgpu_ring_test_ib(ring, tmo);
4841                 if (r)
4842                         return true;
4843         }
4844
4845         for (i = 0; i < adev->gfx.num_compute_rings; i++) {
4846                 ring = &adev->gfx.compute_ring[i];
4847                 r = amdgpu_ring_test_ib(ring, tmo);
4848                 if (r)
4849                         return true;
4850         }
4851
4852         return false;
4853 }
4854
4855 static int gfx_v11_0_post_soft_reset(void *handle)
4856 {
4857         /**
4858          * GFX soft reset will impact MES, need resume MES when do GFX soft reset
4859          */
4860         return amdgpu_mes_resume((struct amdgpu_device *)handle);
4861 }
4862
4863 static uint64_t gfx_v11_0_get_gpu_clock_counter(struct amdgpu_device *adev)
4864 {
4865         uint64_t clock;
4866         uint64_t clock_counter_lo, clock_counter_hi_pre, clock_counter_hi_after;
4867
4868         if (amdgpu_sriov_vf(adev)) {
4869                 amdgpu_gfx_off_ctrl(adev, false);
4870                 mutex_lock(&adev->gfx.gpu_clock_mutex);
4871                 clock_counter_hi_pre = (uint64_t)RREG32_SOC15(GC, 0, regCP_MES_MTIME_HI);
4872                 clock_counter_lo = (uint64_t)RREG32_SOC15(GC, 0, regCP_MES_MTIME_LO);
4873                 clock_counter_hi_after = (uint64_t)RREG32_SOC15(GC, 0, regCP_MES_MTIME_HI);
4874                 if (clock_counter_hi_pre != clock_counter_hi_after)
4875                         clock_counter_lo = (uint64_t)RREG32_SOC15(GC, 0, regCP_MES_MTIME_LO);
4876                 mutex_unlock(&adev->gfx.gpu_clock_mutex);
4877                 amdgpu_gfx_off_ctrl(adev, true);
4878         } else {
4879                 preempt_disable();
4880                 clock_counter_hi_pre = (uint64_t)RREG32_SOC15(SMUIO, 0, regGOLDEN_TSC_COUNT_UPPER);
4881                 clock_counter_lo = (uint64_t)RREG32_SOC15(SMUIO, 0, regGOLDEN_TSC_COUNT_LOWER);
4882                 clock_counter_hi_after = (uint64_t)RREG32_SOC15(SMUIO, 0, regGOLDEN_TSC_COUNT_UPPER);
4883                 if (clock_counter_hi_pre != clock_counter_hi_after)
4884                         clock_counter_lo = (uint64_t)RREG32_SOC15(SMUIO, 0, regGOLDEN_TSC_COUNT_LOWER);
4885                 preempt_enable();
4886         }
4887         clock = clock_counter_lo | (clock_counter_hi_after << 32ULL);
4888
4889         return clock;
4890 }
4891
4892 static void gfx_v11_0_ring_emit_gds_switch(struct amdgpu_ring *ring,
4893                                            uint32_t vmid,
4894                                            uint32_t gds_base, uint32_t gds_size,
4895                                            uint32_t gws_base, uint32_t gws_size,
4896                                            uint32_t oa_base, uint32_t oa_size)
4897 {
4898         struct amdgpu_device *adev = ring->adev;
4899
4900         /* GDS Base */
4901         gfx_v11_0_write_data_to_reg(ring, 0, false,
4902                                     SOC15_REG_OFFSET(GC, 0, regGDS_VMID0_BASE) + 2 * vmid,
4903                                     gds_base);
4904
4905         /* GDS Size */
4906         gfx_v11_0_write_data_to_reg(ring, 0, false,
4907                                     SOC15_REG_OFFSET(GC, 0, regGDS_VMID0_SIZE) + 2 * vmid,
4908                                     gds_size);
4909
4910         /* GWS */
4911         gfx_v11_0_write_data_to_reg(ring, 0, false,
4912                                     SOC15_REG_OFFSET(GC, 0, regGDS_GWS_VMID0) + vmid,
4913                                     gws_size << GDS_GWS_VMID0__SIZE__SHIFT | gws_base);
4914
4915         /* OA */
4916         gfx_v11_0_write_data_to_reg(ring, 0, false,
4917                                     SOC15_REG_OFFSET(GC, 0, regGDS_OA_VMID0) + vmid,
4918                                     (1 << (oa_size + oa_base)) - (1 << oa_base));
4919 }
4920
4921 static int gfx_v11_0_early_init(void *handle)
4922 {
4923         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4924
4925         adev->gfx.funcs = &gfx_v11_0_gfx_funcs;
4926
4927         adev->gfx.num_gfx_rings = GFX11_NUM_GFX_RINGS;
4928         adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev),
4929                                           AMDGPU_MAX_COMPUTE_RINGS);
4930
4931         gfx_v11_0_set_kiq_pm4_funcs(adev);
4932         gfx_v11_0_set_ring_funcs(adev);
4933         gfx_v11_0_set_irq_funcs(adev);
4934         gfx_v11_0_set_gds_init(adev);
4935         gfx_v11_0_set_rlc_funcs(adev);
4936         gfx_v11_0_set_mqd_funcs(adev);
4937         gfx_v11_0_set_imu_funcs(adev);
4938
4939         gfx_v11_0_init_rlcg_reg_access_ctrl(adev);
4940
4941         return gfx_v11_0_init_microcode(adev);
4942 }
4943
4944 static int gfx_v11_0_late_init(void *handle)
4945 {
4946         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4947         int r;
4948
4949         r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0);
4950         if (r)
4951                 return r;
4952
4953         r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0);
4954         if (r)
4955                 return r;
4956
4957         return 0;
4958 }
4959
4960 static bool gfx_v11_0_is_rlc_enabled(struct amdgpu_device *adev)
4961 {
4962         uint32_t rlc_cntl;
4963
4964         /* if RLC is not enabled, do nothing */
4965         rlc_cntl = RREG32_SOC15(GC, 0, regRLC_CNTL);
4966         return (REG_GET_FIELD(rlc_cntl, RLC_CNTL, RLC_ENABLE_F32)) ? true : false;
4967 }
4968
4969 static void gfx_v11_0_set_safe_mode(struct amdgpu_device *adev, int xcc_id)
4970 {
4971         uint32_t data;
4972         unsigned i;
4973
4974         data = RLC_SAFE_MODE__CMD_MASK;
4975         data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT);
4976
4977         WREG32_SOC15(GC, 0, regRLC_SAFE_MODE, data);
4978
4979         /* wait for RLC_SAFE_MODE */
4980         for (i = 0; i < adev->usec_timeout; i++) {
4981                 if (!REG_GET_FIELD(RREG32_SOC15(GC, 0, regRLC_SAFE_MODE),
4982                                    RLC_SAFE_MODE, CMD))
4983                         break;
4984                 udelay(1);
4985         }
4986 }
4987
4988 static void gfx_v11_0_unset_safe_mode(struct amdgpu_device *adev, int xcc_id)
4989 {
4990         WREG32_SOC15(GC, 0, regRLC_SAFE_MODE, RLC_SAFE_MODE__CMD_MASK);
4991 }
4992
4993 static void gfx_v11_0_update_perf_clk(struct amdgpu_device *adev,
4994                                       bool enable)
4995 {
4996         uint32_t def, data;
4997
4998         if (!(adev->cg_flags & AMD_CG_SUPPORT_GFX_PERF_CLK))
4999                 return;
5000
5001         def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
5002
5003         if (enable)
5004                 data &= ~RLC_CGTT_MGCG_OVERRIDE__PERFMON_CLOCK_STATE_MASK;
5005         else
5006                 data |= RLC_CGTT_MGCG_OVERRIDE__PERFMON_CLOCK_STATE_MASK;
5007
5008         if (def != data)
5009                 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data);
5010 }
5011
5012 static void gfx_v11_0_update_sram_fgcg(struct amdgpu_device *adev,
5013                                        bool enable)
5014 {
5015         uint32_t def, data;
5016
5017         if (!(adev->cg_flags & AMD_CG_SUPPORT_GFX_FGCG))
5018                 return;
5019
5020         def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
5021
5022         if (enable)
5023                 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_FGCG_OVERRIDE_MASK;
5024         else
5025                 data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_FGCG_OVERRIDE_MASK;
5026
5027         if (def != data)
5028                 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data);
5029 }
5030
5031 static void gfx_v11_0_update_repeater_fgcg(struct amdgpu_device *adev,
5032                                            bool enable)
5033 {
5034         uint32_t def, data;
5035
5036         if (!(adev->cg_flags & AMD_CG_SUPPORT_REPEATER_FGCG))
5037                 return;
5038
5039         def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
5040
5041         if (enable)
5042                 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_REPEATER_FGCG_OVERRIDE_MASK;
5043         else
5044                 data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_REPEATER_FGCG_OVERRIDE_MASK;
5045
5046         if (def != data)
5047                 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data);
5048 }
5049
5050 static void gfx_v11_0_update_medium_grain_clock_gating(struct amdgpu_device *adev,
5051                                                        bool enable)
5052 {
5053         uint32_t data, def;
5054
5055         if (!(adev->cg_flags & (AMD_CG_SUPPORT_GFX_MGCG | AMD_CG_SUPPORT_GFX_MGLS)))
5056                 return;
5057
5058         /* It is disabled by HW by default */
5059         if (enable) {
5060                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG) {
5061                         /* 1 - RLC_CGTT_MGCG_OVERRIDE */
5062                         def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
5063
5064                         data &= ~(RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
5065                                   RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK |
5066                                   RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK);
5067
5068                         if (def != data)
5069                                 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data);
5070                 }
5071         } else {
5072                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG) {
5073                         def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
5074
5075                         data |= (RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK |
5076                                  RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
5077                                  RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK);
5078
5079                         if (def != data)
5080                                 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data);
5081                 }
5082         }
5083 }
5084
5085 static void gfx_v11_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev,
5086                                                        bool enable)
5087 {
5088         uint32_t def, data;
5089
5090         if (!(adev->cg_flags &
5091               (AMD_CG_SUPPORT_GFX_CGCG |
5092               AMD_CG_SUPPORT_GFX_CGLS |
5093               AMD_CG_SUPPORT_GFX_3D_CGCG |
5094               AMD_CG_SUPPORT_GFX_3D_CGLS)))
5095                 return;
5096
5097         if (enable) {
5098                 def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
5099
5100                 /* unset CGCG override */
5101                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)
5102                         data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGCG_OVERRIDE_MASK;
5103                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
5104                         data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK;
5105                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG ||
5106                     adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS)
5107                         data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_GFX3D_CG_OVERRIDE_MASK;
5108
5109                 /* update CGCG override bits */
5110                 if (def != data)
5111                         WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data);
5112
5113                 /* enable cgcg FSM(0x0000363F) */
5114                 def = data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL);
5115
5116                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG) {
5117                         data &= ~RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD_MASK;
5118                         data |= (0x36 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
5119                                  RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
5120                 }
5121
5122                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) {
5123                         data &= ~RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY_MASK;
5124                         data |= (0x000F << RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
5125                                  RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;
5126                 }
5127
5128                 if (def != data)
5129                         WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL, data);
5130
5131                 /* Program RLC_CGCG_CGLS_CTRL_3D */
5132                 def = data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D);
5133
5134                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG) {
5135                         data &= ~RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD_MASK;
5136                         data |= (0x36 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
5137                                  RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK;
5138                 }
5139
5140                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS) {
5141                         data &= ~RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY_MASK;
5142                         data |= (0xf << RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
5143                                  RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK;
5144                 }
5145
5146                 if (def != data)
5147                         WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D, data);
5148
5149                 /* set IDLE_POLL_COUNT(0x00900100) */
5150                 def = data = RREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_CNTL);
5151
5152                 data &= ~(CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY_MASK | CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK);
5153                 data |= (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) |
5154                         (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
5155
5156                 if (def != data)
5157                         WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_CNTL, data);
5158
5159                 data = RREG32_SOC15(GC, 0, regCP_INT_CNTL);
5160                 data = REG_SET_FIELD(data, CP_INT_CNTL, CNTX_BUSY_INT_ENABLE, 1);
5161                 data = REG_SET_FIELD(data, CP_INT_CNTL, CNTX_EMPTY_INT_ENABLE, 1);
5162                 data = REG_SET_FIELD(data, CP_INT_CNTL, CMP_BUSY_INT_ENABLE, 1);
5163                 data = REG_SET_FIELD(data, CP_INT_CNTL, GFX_IDLE_INT_ENABLE, 1);
5164                 WREG32_SOC15(GC, 0, regCP_INT_CNTL, data);
5165
5166                 data = RREG32_SOC15(GC, 0, regSDMA0_RLC_CGCG_CTRL);
5167                 data = REG_SET_FIELD(data, SDMA0_RLC_CGCG_CTRL, CGCG_INT_ENABLE, 1);
5168                 WREG32_SOC15(GC, 0, regSDMA0_RLC_CGCG_CTRL, data);
5169
5170                 /* Some ASICs only have one SDMA instance, not need to configure SDMA1 */
5171                 if (adev->sdma.num_instances > 1) {
5172                         data = RREG32_SOC15(GC, 0, regSDMA1_RLC_CGCG_CTRL);
5173                         data = REG_SET_FIELD(data, SDMA1_RLC_CGCG_CTRL, CGCG_INT_ENABLE, 1);
5174                         WREG32_SOC15(GC, 0, regSDMA1_RLC_CGCG_CTRL, data);
5175                 }
5176         } else {
5177                 /* Program RLC_CGCG_CGLS_CTRL */
5178                 def = data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL);
5179
5180                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)
5181                         data &= ~RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
5182
5183                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
5184                         data &= ~RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;
5185
5186                 if (def != data)
5187                         WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL, data);
5188
5189                 /* Program RLC_CGCG_CGLS_CTRL_3D */
5190                 def = data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D);
5191
5192                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG)
5193                         data &= ~RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK;
5194                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS)
5195                         data &= ~RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK;
5196
5197                 if (def != data)
5198                         WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D, data);
5199
5200                 data = RREG32_SOC15(GC, 0, regSDMA0_RLC_CGCG_CTRL);
5201                 data &= ~SDMA0_RLC_CGCG_CTRL__CGCG_INT_ENABLE_MASK;
5202                 WREG32_SOC15(GC, 0, regSDMA0_RLC_CGCG_CTRL, data);
5203
5204                 /* Some ASICs only have one SDMA instance, not need to configure SDMA1 */
5205                 if (adev->sdma.num_instances > 1) {
5206                         data = RREG32_SOC15(GC, 0, regSDMA1_RLC_CGCG_CTRL);
5207                         data &= ~SDMA1_RLC_CGCG_CTRL__CGCG_INT_ENABLE_MASK;
5208                         WREG32_SOC15(GC, 0, regSDMA1_RLC_CGCG_CTRL, data);
5209                 }
5210         }
5211 }
5212
5213 static int gfx_v11_0_update_gfx_clock_gating(struct amdgpu_device *adev,
5214                                             bool enable)
5215 {
5216         amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
5217
5218         gfx_v11_0_update_coarse_grain_clock_gating(adev, enable);
5219
5220         gfx_v11_0_update_medium_grain_clock_gating(adev, enable);
5221
5222         gfx_v11_0_update_repeater_fgcg(adev, enable);
5223
5224         gfx_v11_0_update_sram_fgcg(adev, enable);
5225
5226         gfx_v11_0_update_perf_clk(adev, enable);
5227
5228         if (adev->cg_flags &
5229             (AMD_CG_SUPPORT_GFX_MGCG |
5230              AMD_CG_SUPPORT_GFX_CGLS |
5231              AMD_CG_SUPPORT_GFX_CGCG |
5232              AMD_CG_SUPPORT_GFX_3D_CGCG |
5233              AMD_CG_SUPPORT_GFX_3D_CGLS))
5234                 gfx_v11_0_enable_gui_idle_interrupt(adev, enable);
5235
5236         amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
5237
5238         return 0;
5239 }
5240
5241 static void gfx_v11_0_update_spm_vmid(struct amdgpu_device *adev, struct amdgpu_ring *ring, unsigned vmid)
5242 {
5243         u32 reg, pre_data, data;
5244
5245         amdgpu_gfx_off_ctrl(adev, false);
5246         reg = SOC15_REG_OFFSET(GC, 0, regRLC_SPM_MC_CNTL);
5247         if (amdgpu_sriov_is_pp_one_vf(adev) && !amdgpu_sriov_runtime(adev))
5248                 pre_data = RREG32_NO_KIQ(reg);
5249         else
5250                 pre_data = RREG32(reg);
5251
5252         data = pre_data & (~RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK);
5253         data |= (vmid & RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK) << RLC_SPM_MC_CNTL__RLC_SPM_VMID__SHIFT;
5254
5255         if (pre_data != data) {
5256                 if (amdgpu_sriov_is_pp_one_vf(adev) && !amdgpu_sriov_runtime(adev)) {
5257                         WREG32_SOC15_NO_KIQ(GC, 0, regRLC_SPM_MC_CNTL, data);
5258                 } else
5259                         WREG32_SOC15(GC, 0, regRLC_SPM_MC_CNTL, data);
5260         }
5261         amdgpu_gfx_off_ctrl(adev, true);
5262
5263         if (ring
5264                 && amdgpu_sriov_is_pp_one_vf(adev)
5265                 && (pre_data != data)
5266                 && ((ring->funcs->type == AMDGPU_RING_TYPE_GFX)
5267                         || (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE))) {
5268                 amdgpu_ring_emit_wreg(ring, reg, data);
5269         }
5270 }
5271
5272 static const struct amdgpu_rlc_funcs gfx_v11_0_rlc_funcs = {
5273         .is_rlc_enabled = gfx_v11_0_is_rlc_enabled,
5274         .set_safe_mode = gfx_v11_0_set_safe_mode,
5275         .unset_safe_mode = gfx_v11_0_unset_safe_mode,
5276         .init = gfx_v11_0_rlc_init,
5277         .get_csb_size = gfx_v11_0_get_csb_size,
5278         .get_csb_buffer = gfx_v11_0_get_csb_buffer,
5279         .resume = gfx_v11_0_rlc_resume,
5280         .stop = gfx_v11_0_rlc_stop,
5281         .reset = gfx_v11_0_rlc_reset,
5282         .start = gfx_v11_0_rlc_start,
5283         .update_spm_vmid = gfx_v11_0_update_spm_vmid,
5284 };
5285
5286 static void gfx_v11_cntl_power_gating(struct amdgpu_device *adev, bool enable)
5287 {
5288         u32 data = RREG32_SOC15(GC, 0, regRLC_PG_CNTL);
5289
5290         if (enable && (adev->pg_flags & AMD_PG_SUPPORT_GFX_PG))
5291                 data |= RLC_PG_CNTL__GFX_POWER_GATING_ENABLE_MASK;
5292         else
5293                 data &= ~RLC_PG_CNTL__GFX_POWER_GATING_ENABLE_MASK;
5294
5295         WREG32_SOC15(GC, 0, regRLC_PG_CNTL, data);
5296
5297         // Program RLC_PG_DELAY3 for CGPG hysteresis
5298         if (enable && (adev->pg_flags & AMD_PG_SUPPORT_GFX_PG)) {
5299                 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
5300                 case IP_VERSION(11, 0, 1):
5301                 case IP_VERSION(11, 0, 4):
5302                 case IP_VERSION(11, 5, 0):
5303                 case IP_VERSION(11, 5, 1):
5304                 case IP_VERSION(11, 5, 2):
5305                         WREG32_SOC15(GC, 0, regRLC_PG_DELAY_3, RLC_PG_DELAY_3_DEFAULT_GC_11_0_1);
5306                         break;
5307                 default:
5308                         break;
5309                 }
5310         }
5311 }
5312
5313 static void gfx_v11_cntl_pg(struct amdgpu_device *adev, bool enable)
5314 {
5315         amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
5316
5317         gfx_v11_cntl_power_gating(adev, enable);
5318
5319         amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
5320 }
5321
5322 static int gfx_v11_0_set_powergating_state(void *handle,
5323                                            enum amd_powergating_state state)
5324 {
5325         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5326         bool enable = (state == AMD_PG_STATE_GATE);
5327
5328         if (amdgpu_sriov_vf(adev))
5329                 return 0;
5330
5331         switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
5332         case IP_VERSION(11, 0, 0):
5333         case IP_VERSION(11, 0, 2):
5334         case IP_VERSION(11, 0, 3):
5335                 amdgpu_gfx_off_ctrl(adev, enable);
5336                 break;
5337         case IP_VERSION(11, 0, 1):
5338         case IP_VERSION(11, 0, 4):
5339         case IP_VERSION(11, 5, 0):
5340         case IP_VERSION(11, 5, 1):
5341         case IP_VERSION(11, 5, 2):
5342                 if (!enable)
5343                         amdgpu_gfx_off_ctrl(adev, false);
5344
5345                 gfx_v11_cntl_pg(adev, enable);
5346
5347                 if (enable)
5348                         amdgpu_gfx_off_ctrl(adev, true);
5349
5350                 break;
5351         default:
5352                 break;
5353         }
5354
5355         return 0;
5356 }
5357
5358 static int gfx_v11_0_set_clockgating_state(void *handle,
5359                                           enum amd_clockgating_state state)
5360 {
5361         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5362
5363         if (amdgpu_sriov_vf(adev))
5364                 return 0;
5365
5366         switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
5367         case IP_VERSION(11, 0, 0):
5368         case IP_VERSION(11, 0, 1):
5369         case IP_VERSION(11, 0, 2):
5370         case IP_VERSION(11, 0, 3):
5371         case IP_VERSION(11, 0, 4):
5372         case IP_VERSION(11, 5, 0):
5373         case IP_VERSION(11, 5, 1):
5374         case IP_VERSION(11, 5, 2):
5375                 gfx_v11_0_update_gfx_clock_gating(adev,
5376                                 state ==  AMD_CG_STATE_GATE);
5377                 break;
5378         default:
5379                 break;
5380         }
5381
5382         return 0;
5383 }
5384
5385 static void gfx_v11_0_get_clockgating_state(void *handle, u64 *flags)
5386 {
5387         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5388         int data;
5389
5390         /* AMD_CG_SUPPORT_GFX_MGCG */
5391         data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
5392         if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK))
5393                 *flags |= AMD_CG_SUPPORT_GFX_MGCG;
5394
5395         /* AMD_CG_SUPPORT_REPEATER_FGCG */
5396         if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_REPEATER_FGCG_OVERRIDE_MASK))
5397                 *flags |= AMD_CG_SUPPORT_REPEATER_FGCG;
5398
5399         /* AMD_CG_SUPPORT_GFX_FGCG */
5400         if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_FGCG_OVERRIDE_MASK))
5401                 *flags |= AMD_CG_SUPPORT_GFX_FGCG;
5402
5403         /* AMD_CG_SUPPORT_GFX_PERF_CLK */
5404         if (!(data & RLC_CGTT_MGCG_OVERRIDE__PERFMON_CLOCK_STATE_MASK))
5405                 *flags |= AMD_CG_SUPPORT_GFX_PERF_CLK;
5406
5407         /* AMD_CG_SUPPORT_GFX_CGCG */
5408         data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL);
5409         if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK)
5410                 *flags |= AMD_CG_SUPPORT_GFX_CGCG;
5411
5412         /* AMD_CG_SUPPORT_GFX_CGLS */
5413         if (data & RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK)
5414                 *flags |= AMD_CG_SUPPORT_GFX_CGLS;
5415
5416         /* AMD_CG_SUPPORT_GFX_3D_CGCG */
5417         data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D);
5418         if (data & RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK)
5419                 *flags |= AMD_CG_SUPPORT_GFX_3D_CGCG;
5420
5421         /* AMD_CG_SUPPORT_GFX_3D_CGLS */
5422         if (data & RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK)
5423                 *flags |= AMD_CG_SUPPORT_GFX_3D_CGLS;
5424 }
5425
5426 static u64 gfx_v11_0_ring_get_rptr_gfx(struct amdgpu_ring *ring)
5427 {
5428         /* gfx11 is 32bit rptr*/
5429         return *(uint32_t *)ring->rptr_cpu_addr;
5430 }
5431
5432 static u64 gfx_v11_0_ring_get_wptr_gfx(struct amdgpu_ring *ring)
5433 {
5434         struct amdgpu_device *adev = ring->adev;
5435         u64 wptr;
5436
5437         /* XXX check if swapping is necessary on BE */
5438         if (ring->use_doorbell) {
5439                 wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr);
5440         } else {
5441                 wptr = RREG32_SOC15(GC, 0, regCP_RB0_WPTR);
5442                 wptr += (u64)RREG32_SOC15(GC, 0, regCP_RB0_WPTR_HI) << 32;
5443         }
5444
5445         return wptr;
5446 }
5447
5448 static void gfx_v11_0_ring_set_wptr_gfx(struct amdgpu_ring *ring)
5449 {
5450         struct amdgpu_device *adev = ring->adev;
5451
5452         if (ring->use_doorbell) {
5453                 /* XXX check if swapping is necessary on BE */
5454                 atomic64_set((atomic64_t *)ring->wptr_cpu_addr,
5455                              ring->wptr);
5456                 WDOORBELL64(ring->doorbell_index, ring->wptr);
5457         } else {
5458                 WREG32_SOC15(GC, 0, regCP_RB0_WPTR,
5459                              lower_32_bits(ring->wptr));
5460                 WREG32_SOC15(GC, 0, regCP_RB0_WPTR_HI,
5461                              upper_32_bits(ring->wptr));
5462         }
5463 }
5464
5465 static u64 gfx_v11_0_ring_get_rptr_compute(struct amdgpu_ring *ring)
5466 {
5467         /* gfx11 hardware is 32bit rptr */
5468         return *(uint32_t *)ring->rptr_cpu_addr;
5469 }
5470
5471 static u64 gfx_v11_0_ring_get_wptr_compute(struct amdgpu_ring *ring)
5472 {
5473         u64 wptr;
5474
5475         /* XXX check if swapping is necessary on BE */
5476         if (ring->use_doorbell)
5477                 wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr);
5478         else
5479                 BUG();
5480         return wptr;
5481 }
5482
5483 static void gfx_v11_0_ring_set_wptr_compute(struct amdgpu_ring *ring)
5484 {
5485         struct amdgpu_device *adev = ring->adev;
5486
5487         /* XXX check if swapping is necessary on BE */
5488         if (ring->use_doorbell) {
5489                 atomic64_set((atomic64_t *)ring->wptr_cpu_addr,
5490                              ring->wptr);
5491                 WDOORBELL64(ring->doorbell_index, ring->wptr);
5492         } else {
5493                 BUG(); /* only DOORBELL method supported on gfx11 now */
5494         }
5495 }
5496
5497 static void gfx_v11_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
5498 {
5499         struct amdgpu_device *adev = ring->adev;
5500         u32 ref_and_mask, reg_mem_engine;
5501         const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg;
5502
5503         if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
5504                 switch (ring->me) {
5505                 case 1:
5506                         ref_and_mask = nbio_hf_reg->ref_and_mask_cp2 << ring->pipe;
5507                         break;
5508                 case 2:
5509                         ref_and_mask = nbio_hf_reg->ref_and_mask_cp6 << ring->pipe;
5510                         break;
5511                 default:
5512                         return;
5513                 }
5514                 reg_mem_engine = 0;
5515         } else {
5516                 ref_and_mask = nbio_hf_reg->ref_and_mask_cp0 << ring->pipe;
5517                 reg_mem_engine = 1; /* pfp */
5518         }
5519
5520         gfx_v11_0_wait_reg_mem(ring, reg_mem_engine, 0, 1,
5521                                adev->nbio.funcs->get_hdp_flush_req_offset(adev),
5522                                adev->nbio.funcs->get_hdp_flush_done_offset(adev),
5523                                ref_and_mask, ref_and_mask, 0x20);
5524 }
5525
5526 static void gfx_v11_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
5527                                        struct amdgpu_job *job,
5528                                        struct amdgpu_ib *ib,
5529                                        uint32_t flags)
5530 {
5531         unsigned vmid = AMDGPU_JOB_GET_VMID(job);
5532         u32 header, control = 0;
5533
5534         BUG_ON(ib->flags & AMDGPU_IB_FLAG_CE);
5535
5536         header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
5537
5538         control |= ib->length_dw | (vmid << 24);
5539
5540         if (ring->adev->gfx.mcbp && (ib->flags & AMDGPU_IB_FLAG_PREEMPT)) {
5541                 control |= INDIRECT_BUFFER_PRE_ENB(1);
5542
5543                 if (flags & AMDGPU_IB_PREEMPTED)
5544                         control |= INDIRECT_BUFFER_PRE_RESUME(1);
5545
5546                 if (vmid)
5547                         gfx_v11_0_ring_emit_de_meta(ring,
5548                                     (!amdgpu_sriov_vf(ring->adev) && flags & AMDGPU_IB_PREEMPTED) ? true : false);
5549         }
5550
5551         if (ring->is_mes_queue)
5552                 /* inherit vmid from mqd */
5553                 control |= 0x400000;
5554
5555         amdgpu_ring_write(ring, header);
5556         BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
5557         amdgpu_ring_write(ring,
5558 #ifdef __BIG_ENDIAN
5559                 (2 << 0) |
5560 #endif
5561                 lower_32_bits(ib->gpu_addr));
5562         amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
5563         amdgpu_ring_write(ring, control);
5564 }
5565
5566 static void gfx_v11_0_ring_emit_ib_compute(struct amdgpu_ring *ring,
5567                                            struct amdgpu_job *job,
5568                                            struct amdgpu_ib *ib,
5569                                            uint32_t flags)
5570 {
5571         unsigned vmid = AMDGPU_JOB_GET_VMID(job);
5572         u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24);
5573
5574         if (ring->is_mes_queue)
5575                 /* inherit vmid from mqd */
5576                 control |= 0x40000000;
5577
5578         /* Currently, there is a high possibility to get wave ID mismatch
5579          * between ME and GDS, leading to a hw deadlock, because ME generates
5580          * different wave IDs than the GDS expects. This situation happens
5581          * randomly when at least 5 compute pipes use GDS ordered append.
5582          * The wave IDs generated by ME are also wrong after suspend/resume.
5583          * Those are probably bugs somewhere else in the kernel driver.
5584          *
5585          * Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and
5586          * GDS to 0 for this ring (me/pipe).
5587          */
5588         if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) {
5589                 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
5590                 amdgpu_ring_write(ring, regGDS_COMPUTE_MAX_WAVE_ID);
5591                 amdgpu_ring_write(ring, ring->adev->gds.gds_compute_max_wave_id);
5592         }
5593
5594         amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
5595         BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
5596         amdgpu_ring_write(ring,
5597 #ifdef __BIG_ENDIAN
5598                                 (2 << 0) |
5599 #endif
5600                                 lower_32_bits(ib->gpu_addr));
5601         amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
5602         amdgpu_ring_write(ring, control);
5603 }
5604
5605 static void gfx_v11_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
5606                                      u64 seq, unsigned flags)
5607 {
5608         bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
5609         bool int_sel = flags & AMDGPU_FENCE_FLAG_INT;
5610
5611         /* RELEASE_MEM - flush caches, send int */
5612         amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 6));
5613         amdgpu_ring_write(ring, (PACKET3_RELEASE_MEM_GCR_SEQ |
5614                                  PACKET3_RELEASE_MEM_GCR_GL2_WB |
5615                                  PACKET3_RELEASE_MEM_GCR_GLM_INV | /* must be set with GLM_WB */
5616                                  PACKET3_RELEASE_MEM_GCR_GLM_WB |
5617                                  PACKET3_RELEASE_MEM_CACHE_POLICY(3) |
5618                                  PACKET3_RELEASE_MEM_EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
5619                                  PACKET3_RELEASE_MEM_EVENT_INDEX(5)));
5620         amdgpu_ring_write(ring, (PACKET3_RELEASE_MEM_DATA_SEL(write64bit ? 2 : 1) |
5621                                  PACKET3_RELEASE_MEM_INT_SEL(int_sel ? 2 : 0)));
5622
5623         /*
5624          * the address should be Qword aligned if 64bit write, Dword
5625          * aligned if only send 32bit data low (discard data high)
5626          */
5627         if (write64bit)
5628                 BUG_ON(addr & 0x7);
5629         else
5630                 BUG_ON(addr & 0x3);
5631         amdgpu_ring_write(ring, lower_32_bits(addr));
5632         amdgpu_ring_write(ring, upper_32_bits(addr));
5633         amdgpu_ring_write(ring, lower_32_bits(seq));
5634         amdgpu_ring_write(ring, upper_32_bits(seq));
5635         amdgpu_ring_write(ring, ring->is_mes_queue ?
5636                          (ring->hw_queue_id | AMDGPU_FENCE_MES_QUEUE_FLAG) : 0);
5637 }
5638
5639 static void gfx_v11_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
5640 {
5641         int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
5642         uint32_t seq = ring->fence_drv.sync_seq;
5643         uint64_t addr = ring->fence_drv.gpu_addr;
5644
5645         gfx_v11_0_wait_reg_mem(ring, usepfp, 1, 0, lower_32_bits(addr),
5646                                upper_32_bits(addr), seq, 0xffffffff, 4);
5647 }
5648
5649 static void gfx_v11_0_ring_invalidate_tlbs(struct amdgpu_ring *ring,
5650                                    uint16_t pasid, uint32_t flush_type,
5651                                    bool all_hub, uint8_t dst_sel)
5652 {
5653         amdgpu_ring_write(ring, PACKET3(PACKET3_INVALIDATE_TLBS, 0));
5654         amdgpu_ring_write(ring,
5655                           PACKET3_INVALIDATE_TLBS_DST_SEL(dst_sel) |
5656                           PACKET3_INVALIDATE_TLBS_ALL_HUB(all_hub) |
5657                           PACKET3_INVALIDATE_TLBS_PASID(pasid) |
5658                           PACKET3_INVALIDATE_TLBS_FLUSH_TYPE(flush_type));
5659 }
5660
5661 static void gfx_v11_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
5662                                          unsigned vmid, uint64_t pd_addr)
5663 {
5664         if (ring->is_mes_queue)
5665                 gfx_v11_0_ring_invalidate_tlbs(ring, 0, 0, false, 0);
5666         else
5667                 amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
5668
5669         /* compute doesn't have PFP */
5670         if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) {
5671                 /* sync PFP to ME, otherwise we might get invalid PFP reads */
5672                 amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5673                 amdgpu_ring_write(ring, 0x0);
5674         }
5675
5676         /* Make sure that we can't skip the SET_Q_MODE packets when the VM
5677          * changed in any way.
5678          */
5679         ring->set_q_mode_offs = 0;
5680         ring->set_q_mode_ptr = NULL;
5681 }
5682
5683 static void gfx_v11_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr,
5684                                           u64 seq, unsigned int flags)
5685 {
5686         struct amdgpu_device *adev = ring->adev;
5687
5688         /* we only allocate 32bit for each seq wb address */
5689         BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
5690
5691         /* write fence seq to the "addr" */
5692         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5693         amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5694                                  WRITE_DATA_DST_SEL(5) | WR_CONFIRM));
5695         amdgpu_ring_write(ring, lower_32_bits(addr));
5696         amdgpu_ring_write(ring, upper_32_bits(addr));
5697         amdgpu_ring_write(ring, lower_32_bits(seq));
5698
5699         if (flags & AMDGPU_FENCE_FLAG_INT) {
5700                 /* set register to trigger INT */
5701                 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5702                 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5703                                          WRITE_DATA_DST_SEL(0) | WR_CONFIRM));
5704                 amdgpu_ring_write(ring, SOC15_REG_OFFSET(GC, 0, regCPC_INT_STATUS));
5705                 amdgpu_ring_write(ring, 0);
5706                 amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */
5707         }
5708 }
5709
5710 static void gfx_v11_0_ring_emit_cntxcntl(struct amdgpu_ring *ring,
5711                                          uint32_t flags)
5712 {
5713         uint32_t dw2 = 0;
5714
5715         dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */
5716         if (flags & AMDGPU_HAVE_CTX_SWITCH) {
5717                 /* set load_global_config & load_global_uconfig */
5718                 dw2 |= 0x8001;
5719                 /* set load_cs_sh_regs */
5720                 dw2 |= 0x01000000;
5721                 /* set load_per_context_state & load_gfx_sh_regs for GFX */
5722                 dw2 |= 0x10002;
5723         }
5724
5725         amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
5726         amdgpu_ring_write(ring, dw2);
5727         amdgpu_ring_write(ring, 0);
5728 }
5729
5730 static unsigned gfx_v11_0_ring_emit_init_cond_exec(struct amdgpu_ring *ring,
5731                                                    uint64_t addr)
5732 {
5733         unsigned ret;
5734
5735         amdgpu_ring_write(ring, PACKET3(PACKET3_COND_EXEC, 3));
5736         amdgpu_ring_write(ring, lower_32_bits(addr));
5737         amdgpu_ring_write(ring, upper_32_bits(addr));
5738         /* discard following DWs if *cond_exec_gpu_addr==0 */
5739         amdgpu_ring_write(ring, 0);
5740         ret = ring->wptr & ring->buf_mask;
5741         /* patch dummy value later */
5742         amdgpu_ring_write(ring, 0);
5743
5744         return ret;
5745 }
5746
5747 static void gfx_v11_0_ring_emit_gfx_shadow(struct amdgpu_ring *ring,
5748                                            u64 shadow_va, u64 csa_va,
5749                                            u64 gds_va, bool init_shadow,
5750                                            int vmid)
5751 {
5752         struct amdgpu_device *adev = ring->adev;
5753         unsigned int offs, end;
5754
5755         if (!adev->gfx.cp_gfx_shadow || !ring->ring_obj)
5756                 return;
5757
5758         /*
5759          * The logic here isn't easy to understand because we need to keep state
5760          * accross multiple executions of the function as well as between the
5761          * CPU and GPU. The general idea is that the newly written GPU command
5762          * has a condition on the previous one and only executed if really
5763          * necessary.
5764          */
5765
5766         /*
5767          * The dw in the NOP controls if the next SET_Q_MODE packet should be
5768          * executed or not. Reserve 64bits just to be on the save side.
5769          */
5770         amdgpu_ring_write(ring, PACKET3(PACKET3_NOP, 1));
5771         offs = ring->wptr & ring->buf_mask;
5772
5773         /*
5774          * We start with skipping the prefix SET_Q_MODE and always executing
5775          * the postfix SET_Q_MODE packet. This is changed below with a
5776          * WRITE_DATA command when the postfix executed.
5777          */
5778         amdgpu_ring_write(ring, shadow_va ? 1 : 0);
5779         amdgpu_ring_write(ring, 0);
5780
5781         if (ring->set_q_mode_offs) {
5782                 uint64_t addr;
5783
5784                 addr = amdgpu_bo_gpu_offset(ring->ring_obj);
5785                 addr += ring->set_q_mode_offs << 2;
5786                 end = gfx_v11_0_ring_emit_init_cond_exec(ring, addr);
5787         }
5788
5789         /*
5790          * When the postfix SET_Q_MODE packet executes we need to make sure that the
5791          * next prefix SET_Q_MODE packet executes as well.
5792          */
5793         if (!shadow_va) {
5794                 uint64_t addr;
5795
5796                 addr = amdgpu_bo_gpu_offset(ring->ring_obj);
5797                 addr += offs << 2;
5798                 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5799                 amdgpu_ring_write(ring, WRITE_DATA_DST_SEL(5) | WR_CONFIRM);
5800                 amdgpu_ring_write(ring, lower_32_bits(addr));
5801                 amdgpu_ring_write(ring, upper_32_bits(addr));
5802                 amdgpu_ring_write(ring, 0x1);
5803         }
5804
5805         amdgpu_ring_write(ring, PACKET3(PACKET3_SET_Q_PREEMPTION_MODE, 7));
5806         amdgpu_ring_write(ring, lower_32_bits(shadow_va));
5807         amdgpu_ring_write(ring, upper_32_bits(shadow_va));
5808         amdgpu_ring_write(ring, lower_32_bits(gds_va));
5809         amdgpu_ring_write(ring, upper_32_bits(gds_va));
5810         amdgpu_ring_write(ring, lower_32_bits(csa_va));
5811         amdgpu_ring_write(ring, upper_32_bits(csa_va));
5812         amdgpu_ring_write(ring, shadow_va ?
5813                           PACKET3_SET_Q_PREEMPTION_MODE_IB_VMID(vmid) : 0);
5814         amdgpu_ring_write(ring, init_shadow ?
5815                           PACKET3_SET_Q_PREEMPTION_MODE_INIT_SHADOW_MEM : 0);
5816
5817         if (ring->set_q_mode_offs)
5818                 amdgpu_ring_patch_cond_exec(ring, end);
5819
5820         if (shadow_va) {
5821                 uint64_t token = shadow_va ^ csa_va ^ gds_va ^ vmid;
5822
5823                 /*
5824                  * If the tokens match try to skip the last postfix SET_Q_MODE
5825                  * packet to avoid saving/restoring the state all the time.
5826                  */
5827                 if (ring->set_q_mode_ptr && ring->set_q_mode_token == token)
5828                         *ring->set_q_mode_ptr = 0;
5829
5830                 ring->set_q_mode_token = token;
5831         } else {
5832                 ring->set_q_mode_ptr = &ring->ring[ring->set_q_mode_offs];
5833         }
5834
5835         ring->set_q_mode_offs = offs;
5836 }
5837
5838 static int gfx_v11_0_ring_preempt_ib(struct amdgpu_ring *ring)
5839 {
5840         int i, r = 0;
5841         struct amdgpu_device *adev = ring->adev;
5842         struct amdgpu_kiq *kiq = &adev->gfx.kiq[0];
5843         struct amdgpu_ring *kiq_ring = &kiq->ring;
5844         unsigned long flags;
5845
5846         if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues)
5847                 return -EINVAL;
5848
5849         spin_lock_irqsave(&kiq->ring_lock, flags);
5850
5851         if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size)) {
5852                 spin_unlock_irqrestore(&kiq->ring_lock, flags);
5853                 return -ENOMEM;
5854         }
5855
5856         /* assert preemption condition */
5857         amdgpu_ring_set_preempt_cond_exec(ring, false);
5858
5859         /* assert IB preemption, emit the trailing fence */
5860         kiq->pmf->kiq_unmap_queues(kiq_ring, ring, PREEMPT_QUEUES_NO_UNMAP,
5861                                    ring->trail_fence_gpu_addr,
5862                                    ++ring->trail_seq);
5863         amdgpu_ring_commit(kiq_ring);
5864
5865         spin_unlock_irqrestore(&kiq->ring_lock, flags);
5866
5867         /* poll the trailing fence */
5868         for (i = 0; i < adev->usec_timeout; i++) {
5869                 if (ring->trail_seq ==
5870                     le32_to_cpu(*(ring->trail_fence_cpu_addr)))
5871                         break;
5872                 udelay(1);
5873         }
5874
5875         if (i >= adev->usec_timeout) {
5876                 r = -EINVAL;
5877                 DRM_ERROR("ring %d failed to preempt ib\n", ring->idx);
5878         }
5879
5880         /* deassert preemption condition */
5881         amdgpu_ring_set_preempt_cond_exec(ring, true);
5882         return r;
5883 }
5884
5885 static void gfx_v11_0_ring_emit_de_meta(struct amdgpu_ring *ring, bool resume)
5886 {
5887         struct amdgpu_device *adev = ring->adev;
5888         struct v10_de_ib_state de_payload = {0};
5889         uint64_t offset, gds_addr, de_payload_gpu_addr;
5890         void *de_payload_cpu_addr;
5891         int cnt;
5892
5893         if (ring->is_mes_queue) {
5894                 offset = offsetof(struct amdgpu_mes_ctx_meta_data,
5895                                   gfx[0].gfx_meta_data) +
5896                         offsetof(struct v10_gfx_meta_data, de_payload);
5897                 de_payload_gpu_addr =
5898                         amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset);
5899                 de_payload_cpu_addr =
5900                         amdgpu_mes_ctx_get_offs_cpu_addr(ring, offset);
5901
5902                 offset = offsetof(struct amdgpu_mes_ctx_meta_data,
5903                                   gfx[0].gds_backup) +
5904                         offsetof(struct v10_gfx_meta_data, de_payload);
5905                 gds_addr = amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset);
5906         } else {
5907                 offset = offsetof(struct v10_gfx_meta_data, de_payload);
5908                 de_payload_gpu_addr = amdgpu_csa_vaddr(ring->adev) + offset;
5909                 de_payload_cpu_addr = adev->virt.csa_cpu_addr + offset;
5910
5911                 gds_addr = ALIGN(amdgpu_csa_vaddr(ring->adev) +
5912                                  AMDGPU_CSA_SIZE - adev->gds.gds_size,
5913                                  PAGE_SIZE);
5914         }
5915
5916         de_payload.gds_backup_addrlo = lower_32_bits(gds_addr);
5917         de_payload.gds_backup_addrhi = upper_32_bits(gds_addr);
5918
5919         cnt = (sizeof(de_payload) >> 2) + 4 - 2;
5920         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt));
5921         amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
5922                                  WRITE_DATA_DST_SEL(8) |
5923                                  WR_CONFIRM) |
5924                                  WRITE_DATA_CACHE_POLICY(0));
5925         amdgpu_ring_write(ring, lower_32_bits(de_payload_gpu_addr));
5926         amdgpu_ring_write(ring, upper_32_bits(de_payload_gpu_addr));
5927
5928         if (resume)
5929                 amdgpu_ring_write_multiple(ring, de_payload_cpu_addr,
5930                                            sizeof(de_payload) >> 2);
5931         else
5932                 amdgpu_ring_write_multiple(ring, (void *)&de_payload,
5933                                            sizeof(de_payload) >> 2);
5934 }
5935
5936 static void gfx_v11_0_ring_emit_frame_cntl(struct amdgpu_ring *ring, bool start,
5937                                     bool secure)
5938 {
5939         uint32_t v = secure ? FRAME_TMZ : 0;
5940
5941         amdgpu_ring_write(ring, PACKET3(PACKET3_FRAME_CONTROL, 0));
5942         amdgpu_ring_write(ring, v | FRAME_CMD(start ? 0 : 1));
5943 }
5944
5945 static void gfx_v11_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg,
5946                                      uint32_t reg_val_offs)
5947 {
5948         struct amdgpu_device *adev = ring->adev;
5949
5950         amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));
5951         amdgpu_ring_write(ring, 0 |     /* src: register*/
5952                                 (5 << 8) |      /* dst: memory */
5953                                 (1 << 20));     /* write confirm */
5954         amdgpu_ring_write(ring, reg);
5955         amdgpu_ring_write(ring, 0);
5956         amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +
5957                                 reg_val_offs * 4));
5958         amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +
5959                                 reg_val_offs * 4));
5960 }
5961
5962 static void gfx_v11_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg,
5963                                    uint32_t val)
5964 {
5965         uint32_t cmd = 0;
5966
5967         switch (ring->funcs->type) {
5968         case AMDGPU_RING_TYPE_GFX:
5969                 cmd = WRITE_DATA_ENGINE_SEL(1) | WR_CONFIRM;
5970                 break;
5971         case AMDGPU_RING_TYPE_KIQ:
5972                 cmd = (1 << 16); /* no inc addr */
5973                 break;
5974         default:
5975                 cmd = WR_CONFIRM;
5976                 break;
5977         }
5978         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5979         amdgpu_ring_write(ring, cmd);
5980         amdgpu_ring_write(ring, reg);
5981         amdgpu_ring_write(ring, 0);
5982         amdgpu_ring_write(ring, val);
5983 }
5984
5985 static void gfx_v11_0_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
5986                                         uint32_t val, uint32_t mask)
5987 {
5988         gfx_v11_0_wait_reg_mem(ring, 0, 0, 0, reg, 0, val, mask, 0x20);
5989 }
5990
5991 static void gfx_v11_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring,
5992                                                    uint32_t reg0, uint32_t reg1,
5993                                                    uint32_t ref, uint32_t mask)
5994 {
5995         int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
5996
5997         gfx_v11_0_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1,
5998                                ref, mask, 0x20);
5999 }
6000
6001 static void gfx_v11_0_ring_soft_recovery(struct amdgpu_ring *ring,
6002                                          unsigned vmid)
6003 {
6004         struct amdgpu_device *adev = ring->adev;
6005         uint32_t value = 0;
6006
6007         value = REG_SET_FIELD(value, SQ_CMD, CMD, 0x03);
6008         value = REG_SET_FIELD(value, SQ_CMD, MODE, 0x01);
6009         value = REG_SET_FIELD(value, SQ_CMD, CHECK_VMID, 1);
6010         value = REG_SET_FIELD(value, SQ_CMD, VM_ID, vmid);
6011         WREG32_SOC15(GC, 0, regSQ_CMD, value);
6012 }
6013
6014 static void
6015 gfx_v11_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev,
6016                                       uint32_t me, uint32_t pipe,
6017                                       enum amdgpu_interrupt_state state)
6018 {
6019         uint32_t cp_int_cntl, cp_int_cntl_reg;
6020
6021         if (!me) {
6022                 switch (pipe) {
6023                 case 0:
6024                         cp_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_INT_CNTL_RING0);
6025                         break;
6026                 case 1:
6027                         cp_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_INT_CNTL_RING1);
6028                         break;
6029                 default:
6030                         DRM_DEBUG("invalid pipe %d\n", pipe);
6031                         return;
6032                 }
6033         } else {
6034                 DRM_DEBUG("invalid me %d\n", me);
6035                 return;
6036         }
6037
6038         switch (state) {
6039         case AMDGPU_IRQ_STATE_DISABLE:
6040                 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg);
6041                 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
6042                                             TIME_STAMP_INT_ENABLE, 0);
6043                 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
6044                                             GENERIC0_INT_ENABLE, 0);
6045                 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl);
6046                 break;
6047         case AMDGPU_IRQ_STATE_ENABLE:
6048                 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg);
6049                 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
6050                                             TIME_STAMP_INT_ENABLE, 1);
6051                 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
6052                                             GENERIC0_INT_ENABLE, 1);
6053                 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl);
6054                 break;
6055         default:
6056                 break;
6057         }
6058 }
6059
6060 static void gfx_v11_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev,
6061                                                      int me, int pipe,
6062                                                      enum amdgpu_interrupt_state state)
6063 {
6064         u32 mec_int_cntl, mec_int_cntl_reg;
6065
6066         /*
6067          * amdgpu controls only the first MEC. That's why this function only
6068          * handles the setting of interrupts for this specific MEC. All other
6069          * pipes' interrupts are set by amdkfd.
6070          */
6071
6072         if (me == 1) {
6073                 switch (pipe) {
6074                 case 0:
6075                         mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE0_INT_CNTL);
6076                         break;
6077                 case 1:
6078                         mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE1_INT_CNTL);
6079                         break;
6080                 case 2:
6081                         mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE2_INT_CNTL);
6082                         break;
6083                 case 3:
6084                         mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE3_INT_CNTL);
6085                         break;
6086                 default:
6087                         DRM_DEBUG("invalid pipe %d\n", pipe);
6088                         return;
6089                 }
6090         } else {
6091                 DRM_DEBUG("invalid me %d\n", me);
6092                 return;
6093         }
6094
6095         switch (state) {
6096         case AMDGPU_IRQ_STATE_DISABLE:
6097                 mec_int_cntl = RREG32_SOC15_IP(GC, mec_int_cntl_reg);
6098                 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
6099                                              TIME_STAMP_INT_ENABLE, 0);
6100                 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
6101                                              GENERIC0_INT_ENABLE, 0);
6102                 WREG32_SOC15_IP(GC, mec_int_cntl_reg, mec_int_cntl);
6103                 break;
6104         case AMDGPU_IRQ_STATE_ENABLE:
6105                 mec_int_cntl = RREG32_SOC15_IP(GC, mec_int_cntl_reg);
6106                 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
6107                                              TIME_STAMP_INT_ENABLE, 1);
6108                 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
6109                                              GENERIC0_INT_ENABLE, 1);
6110                 WREG32_SOC15_IP(GC, mec_int_cntl_reg, mec_int_cntl);
6111                 break;
6112         default:
6113                 break;
6114         }
6115 }
6116
6117 static int gfx_v11_0_set_eop_interrupt_state(struct amdgpu_device *adev,
6118                                             struct amdgpu_irq_src *src,
6119                                             unsigned type,
6120                                             enum amdgpu_interrupt_state state)
6121 {
6122         switch (type) {
6123         case AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP:
6124                 gfx_v11_0_set_gfx_eop_interrupt_state(adev, 0, 0, state);
6125                 break;
6126         case AMDGPU_CP_IRQ_GFX_ME0_PIPE1_EOP:
6127                 gfx_v11_0_set_gfx_eop_interrupt_state(adev, 0, 1, state);
6128                 break;
6129         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP:
6130                 gfx_v11_0_set_compute_eop_interrupt_state(adev, 1, 0, state);
6131                 break;
6132         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP:
6133                 gfx_v11_0_set_compute_eop_interrupt_state(adev, 1, 1, state);
6134                 break;
6135         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP:
6136                 gfx_v11_0_set_compute_eop_interrupt_state(adev, 1, 2, state);
6137                 break;
6138         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP:
6139                 gfx_v11_0_set_compute_eop_interrupt_state(adev, 1, 3, state);
6140                 break;
6141         default:
6142                 break;
6143         }
6144         return 0;
6145 }
6146
6147 static int gfx_v11_0_eop_irq(struct amdgpu_device *adev,
6148                              struct amdgpu_irq_src *source,
6149                              struct amdgpu_iv_entry *entry)
6150 {
6151         int i;
6152         u8 me_id, pipe_id, queue_id;
6153         struct amdgpu_ring *ring;
6154         uint32_t mes_queue_id = entry->src_data[0];
6155
6156         DRM_DEBUG("IH: CP EOP\n");
6157
6158         if (adev->enable_mes && (mes_queue_id & AMDGPU_FENCE_MES_QUEUE_FLAG)) {
6159                 struct amdgpu_mes_queue *queue;
6160
6161                 mes_queue_id &= AMDGPU_FENCE_MES_QUEUE_ID_MASK;
6162
6163                 spin_lock(&adev->mes.queue_id_lock);
6164                 queue = idr_find(&adev->mes.queue_id_idr, mes_queue_id);
6165                 if (queue) {
6166                         DRM_DEBUG("process mes queue id = %d\n", mes_queue_id);
6167                         amdgpu_fence_process(queue->ring);
6168                 }
6169                 spin_unlock(&adev->mes.queue_id_lock);
6170         } else {
6171                 me_id = (entry->ring_id & 0x0c) >> 2;
6172                 pipe_id = (entry->ring_id & 0x03) >> 0;
6173                 queue_id = (entry->ring_id & 0x70) >> 4;
6174
6175                 switch (me_id) {
6176                 case 0:
6177                         if (pipe_id == 0)
6178                                 amdgpu_fence_process(&adev->gfx.gfx_ring[0]);
6179                         else
6180                                 amdgpu_fence_process(&adev->gfx.gfx_ring[1]);
6181                         break;
6182                 case 1:
6183                 case 2:
6184                         for (i = 0; i < adev->gfx.num_compute_rings; i++) {
6185                                 ring = &adev->gfx.compute_ring[i];
6186                                 /* Per-queue interrupt is supported for MEC starting from VI.
6187                                  * The interrupt can only be enabled/disabled per pipe instead
6188                                  * of per queue.
6189                                  */
6190                                 if ((ring->me == me_id) &&
6191                                     (ring->pipe == pipe_id) &&
6192                                     (ring->queue == queue_id))
6193                                         amdgpu_fence_process(ring);
6194                         }
6195                         break;
6196                 }
6197         }
6198
6199         return 0;
6200 }
6201
6202 static int gfx_v11_0_set_priv_reg_fault_state(struct amdgpu_device *adev,
6203                                               struct amdgpu_irq_src *source,
6204                                               unsigned type,
6205                                               enum amdgpu_interrupt_state state)
6206 {
6207         switch (state) {
6208         case AMDGPU_IRQ_STATE_DISABLE:
6209         case AMDGPU_IRQ_STATE_ENABLE:
6210                 WREG32_FIELD15_PREREG(GC, 0, CP_INT_CNTL_RING0,
6211                                PRIV_REG_INT_ENABLE,
6212                                state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
6213                 break;
6214         default:
6215                 break;
6216         }
6217
6218         return 0;
6219 }
6220
6221 static int gfx_v11_0_set_priv_inst_fault_state(struct amdgpu_device *adev,
6222                                                struct amdgpu_irq_src *source,
6223                                                unsigned type,
6224                                                enum amdgpu_interrupt_state state)
6225 {
6226         switch (state) {
6227         case AMDGPU_IRQ_STATE_DISABLE:
6228         case AMDGPU_IRQ_STATE_ENABLE:
6229                 WREG32_FIELD15_PREREG(GC, 0, CP_INT_CNTL_RING0,
6230                                PRIV_INSTR_INT_ENABLE,
6231                                state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
6232                 break;
6233         default:
6234                 break;
6235         }
6236
6237         return 0;
6238 }
6239
6240 static void gfx_v11_0_handle_priv_fault(struct amdgpu_device *adev,
6241                                         struct amdgpu_iv_entry *entry)
6242 {
6243         u8 me_id, pipe_id, queue_id;
6244         struct amdgpu_ring *ring;
6245         int i;
6246
6247         me_id = (entry->ring_id & 0x0c) >> 2;
6248         pipe_id = (entry->ring_id & 0x03) >> 0;
6249         queue_id = (entry->ring_id & 0x70) >> 4;
6250
6251         switch (me_id) {
6252         case 0:
6253                 for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
6254                         ring = &adev->gfx.gfx_ring[i];
6255                         /* we only enabled 1 gfx queue per pipe for now */
6256                         if (ring->me == me_id && ring->pipe == pipe_id)
6257                                 drm_sched_fault(&ring->sched);
6258                 }
6259                 break;
6260         case 1:
6261         case 2:
6262                 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
6263                         ring = &adev->gfx.compute_ring[i];
6264                         if (ring->me == me_id && ring->pipe == pipe_id &&
6265                             ring->queue == queue_id)
6266                                 drm_sched_fault(&ring->sched);
6267                 }
6268                 break;
6269         default:
6270                 BUG();
6271                 break;
6272         }
6273 }
6274
6275 static int gfx_v11_0_priv_reg_irq(struct amdgpu_device *adev,
6276                                   struct amdgpu_irq_src *source,
6277                                   struct amdgpu_iv_entry *entry)
6278 {
6279         DRM_ERROR("Illegal register access in command stream\n");
6280         gfx_v11_0_handle_priv_fault(adev, entry);
6281         return 0;
6282 }
6283
6284 static int gfx_v11_0_priv_inst_irq(struct amdgpu_device *adev,
6285                                    struct amdgpu_irq_src *source,
6286                                    struct amdgpu_iv_entry *entry)
6287 {
6288         DRM_ERROR("Illegal instruction in command stream\n");
6289         gfx_v11_0_handle_priv_fault(adev, entry);
6290         return 0;
6291 }
6292
6293 static int gfx_v11_0_rlc_gc_fed_irq(struct amdgpu_device *adev,
6294                                   struct amdgpu_irq_src *source,
6295                                   struct amdgpu_iv_entry *entry)
6296 {
6297         if (adev->gfx.ras && adev->gfx.ras->rlc_gc_fed_irq)
6298                 return adev->gfx.ras->rlc_gc_fed_irq(adev, source, entry);
6299
6300         return 0;
6301 }
6302
6303 #if 0
6304 static int gfx_v11_0_kiq_set_interrupt_state(struct amdgpu_device *adev,
6305                                              struct amdgpu_irq_src *src,
6306                                              unsigned int type,
6307                                              enum amdgpu_interrupt_state state)
6308 {
6309         uint32_t tmp, target;
6310         struct amdgpu_ring *ring = &(adev->gfx.kiq[0].ring);
6311
6312         target = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE0_INT_CNTL);
6313         target += ring->pipe;
6314
6315         switch (type) {
6316         case AMDGPU_CP_KIQ_IRQ_DRIVER0:
6317                 if (state == AMDGPU_IRQ_STATE_DISABLE) {
6318                         tmp = RREG32_SOC15(GC, 0, regCPC_INT_CNTL);
6319                         tmp = REG_SET_FIELD(tmp, CPC_INT_CNTL,
6320                                             GENERIC2_INT_ENABLE, 0);
6321                         WREG32_SOC15(GC, 0, regCPC_INT_CNTL, tmp);
6322
6323                         tmp = RREG32_SOC15_IP(GC, target);
6324                         tmp = REG_SET_FIELD(tmp, CP_ME1_PIPE0_INT_CNTL,
6325                                             GENERIC2_INT_ENABLE, 0);
6326                         WREG32_SOC15_IP(GC, target, tmp);
6327                 } else {
6328                         tmp = RREG32_SOC15(GC, 0, regCPC_INT_CNTL);
6329                         tmp = REG_SET_FIELD(tmp, CPC_INT_CNTL,
6330                                             GENERIC2_INT_ENABLE, 1);
6331                         WREG32_SOC15(GC, 0, regCPC_INT_CNTL, tmp);
6332
6333                         tmp = RREG32_SOC15_IP(GC, target);
6334                         tmp = REG_SET_FIELD(tmp, CP_ME1_PIPE0_INT_CNTL,
6335                                             GENERIC2_INT_ENABLE, 1);
6336                         WREG32_SOC15_IP(GC, target, tmp);
6337                 }
6338                 break;
6339         default:
6340                 BUG(); /* kiq only support GENERIC2_INT now */
6341                 break;
6342         }
6343         return 0;
6344 }
6345 #endif
6346
6347 static void gfx_v11_0_emit_mem_sync(struct amdgpu_ring *ring)
6348 {
6349         const unsigned int gcr_cntl =
6350                         PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_INV(1) |
6351                         PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_WB(1) |
6352                         PACKET3_ACQUIRE_MEM_GCR_CNTL_GLM_INV(1) |
6353                         PACKET3_ACQUIRE_MEM_GCR_CNTL_GLM_WB(1) |
6354                         PACKET3_ACQUIRE_MEM_GCR_CNTL_GL1_INV(1) |
6355                         PACKET3_ACQUIRE_MEM_GCR_CNTL_GLV_INV(1) |
6356                         PACKET3_ACQUIRE_MEM_GCR_CNTL_GLK_INV(1) |
6357                         PACKET3_ACQUIRE_MEM_GCR_CNTL_GLI_INV(1);
6358
6359         /* ACQUIRE_MEM - make one or more surfaces valid for use by the subsequent operations */
6360         amdgpu_ring_write(ring, PACKET3(PACKET3_ACQUIRE_MEM, 6));
6361         amdgpu_ring_write(ring, 0); /* CP_COHER_CNTL */
6362         amdgpu_ring_write(ring, 0xffffffff);  /* CP_COHER_SIZE */
6363         amdgpu_ring_write(ring, 0xffffff);  /* CP_COHER_SIZE_HI */
6364         amdgpu_ring_write(ring, 0); /* CP_COHER_BASE */
6365         amdgpu_ring_write(ring, 0);  /* CP_COHER_BASE_HI */
6366         amdgpu_ring_write(ring, 0x0000000A); /* POLL_INTERVAL */
6367         amdgpu_ring_write(ring, gcr_cntl); /* GCR_CNTL */
6368 }
6369
6370 static void gfx_v11_ip_print(void *handle, struct drm_printer *p)
6371 {
6372         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
6373         uint32_t i, j, k, reg, index = 0;
6374         uint32_t reg_count = ARRAY_SIZE(gc_reg_list_11_0);
6375
6376         if (!adev->gfx.ip_dump_core)
6377                 return;
6378
6379         for (i = 0; i < reg_count; i++)
6380                 drm_printf(p, "%-50s \t 0x%08x\n",
6381                            gc_reg_list_11_0[i].reg_name,
6382                            adev->gfx.ip_dump_core[i]);
6383
6384         /* print compute queue registers for all instances */
6385         if (!adev->gfx.ip_dump_compute_queues)
6386                 return;
6387
6388         reg_count = ARRAY_SIZE(gc_cp_reg_list_11);
6389         drm_printf(p, "\nnum_mec: %d num_pipe: %d num_queue: %d\n",
6390                    adev->gfx.mec.num_mec,
6391                    adev->gfx.mec.num_pipe_per_mec,
6392                    adev->gfx.mec.num_queue_per_pipe);
6393
6394         for (i = 0; i < adev->gfx.mec.num_mec; i++) {
6395                 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) {
6396                         for (k = 0; k < adev->gfx.mec.num_queue_per_pipe; k++) {
6397                                 drm_printf(p, "\nmec %d, pipe %d, queue %d\n", i, j, k);
6398                                 for (reg = 0; reg < reg_count; reg++) {
6399                                         drm_printf(p, "%-50s \t 0x%08x\n",
6400                                                    gc_cp_reg_list_11[reg].reg_name,
6401                                                    adev->gfx.ip_dump_compute_queues[index + reg]);
6402                                 }
6403                                 index += reg_count;
6404                         }
6405                 }
6406         }
6407
6408         /* print gfx queue registers for all instances */
6409         if (!adev->gfx.ip_dump_gfx_queues)
6410                 return;
6411
6412         index = 0;
6413         reg_count = ARRAY_SIZE(gc_gfx_queue_reg_list_11);
6414         drm_printf(p, "\nnum_me: %d num_pipe: %d num_queue: %d\n",
6415                    adev->gfx.me.num_me,
6416                    adev->gfx.me.num_pipe_per_me,
6417                    adev->gfx.me.num_queue_per_pipe);
6418
6419         for (i = 0; i < adev->gfx.me.num_me; i++) {
6420                 for (j = 0; j < adev->gfx.me.num_pipe_per_me; j++) {
6421                         for (k = 0; k < adev->gfx.me.num_queue_per_pipe; k++) {
6422                                 drm_printf(p, "\nme %d, pipe %d, queue %d\n", i, j, k);
6423                                 for (reg = 0; reg < reg_count; reg++) {
6424                                         drm_printf(p, "%-50s \t 0x%08x\n",
6425                                                    gc_gfx_queue_reg_list_11[reg].reg_name,
6426                                                    adev->gfx.ip_dump_gfx_queues[index + reg]);
6427                                 }
6428                                 index += reg_count;
6429                         }
6430                 }
6431         }
6432 }
6433
6434 static void gfx_v11_ip_dump(void *handle)
6435 {
6436         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
6437         uint32_t i, j, k, reg, index = 0;
6438         uint32_t reg_count = ARRAY_SIZE(gc_reg_list_11_0);
6439
6440         if (!adev->gfx.ip_dump_core)
6441                 return;
6442
6443         amdgpu_gfx_off_ctrl(adev, false);
6444         for (i = 0; i < reg_count; i++)
6445                 adev->gfx.ip_dump_core[i] = RREG32(SOC15_REG_ENTRY_OFFSET(gc_reg_list_11_0[i]));
6446         amdgpu_gfx_off_ctrl(adev, true);
6447
6448         /* dump compute queue registers for all instances */
6449         if (!adev->gfx.ip_dump_compute_queues)
6450                 return;
6451
6452         reg_count = ARRAY_SIZE(gc_cp_reg_list_11);
6453         amdgpu_gfx_off_ctrl(adev, false);
6454         mutex_lock(&adev->srbm_mutex);
6455         for (i = 0; i < adev->gfx.mec.num_mec; i++) {
6456                 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) {
6457                         for (k = 0; k < adev->gfx.mec.num_queue_per_pipe; k++) {
6458                                 /* ME0 is for GFX so start from 1 for CP */
6459                                 soc21_grbm_select(adev, adev->gfx.me.num_me + i, j, k, 0);
6460                                 for (reg = 0; reg < reg_count; reg++) {
6461                                         adev->gfx.ip_dump_compute_queues[index + reg] =
6462                                                 RREG32(SOC15_REG_ENTRY_OFFSET(
6463                                                         gc_cp_reg_list_11[reg]));
6464                                 }
6465                                 index += reg_count;
6466                         }
6467                 }
6468         }
6469         soc21_grbm_select(adev, 0, 0, 0, 0);
6470         mutex_unlock(&adev->srbm_mutex);
6471         amdgpu_gfx_off_ctrl(adev, true);
6472
6473         /* dump gfx queue registers for all instances */
6474         if (!adev->gfx.ip_dump_gfx_queues)
6475                 return;
6476
6477         index = 0;
6478         reg_count = ARRAY_SIZE(gc_gfx_queue_reg_list_11);
6479         amdgpu_gfx_off_ctrl(adev, false);
6480         mutex_lock(&adev->srbm_mutex);
6481         for (i = 0; i < adev->gfx.me.num_me; i++) {
6482                 for (j = 0; j < adev->gfx.me.num_pipe_per_me; j++) {
6483                         for (k = 0; k < adev->gfx.me.num_queue_per_pipe; k++) {
6484                                 soc21_grbm_select(adev, i, j, k, 0);
6485
6486                                 for (reg = 0; reg < reg_count; reg++) {
6487                                         adev->gfx.ip_dump_gfx_queues[index + reg] =
6488                                                 RREG32(SOC15_REG_ENTRY_OFFSET(
6489                                                         gc_gfx_queue_reg_list_11[reg]));
6490                                 }
6491                                 index += reg_count;
6492                         }
6493                 }
6494         }
6495         soc21_grbm_select(adev, 0, 0, 0, 0);
6496         mutex_unlock(&adev->srbm_mutex);
6497         amdgpu_gfx_off_ctrl(adev, true);
6498 }
6499
6500 static const struct amd_ip_funcs gfx_v11_0_ip_funcs = {
6501         .name = "gfx_v11_0",
6502         .early_init = gfx_v11_0_early_init,
6503         .late_init = gfx_v11_0_late_init,
6504         .sw_init = gfx_v11_0_sw_init,
6505         .sw_fini = gfx_v11_0_sw_fini,
6506         .hw_init = gfx_v11_0_hw_init,
6507         .hw_fini = gfx_v11_0_hw_fini,
6508         .suspend = gfx_v11_0_suspend,
6509         .resume = gfx_v11_0_resume,
6510         .is_idle = gfx_v11_0_is_idle,
6511         .wait_for_idle = gfx_v11_0_wait_for_idle,
6512         .soft_reset = gfx_v11_0_soft_reset,
6513         .check_soft_reset = gfx_v11_0_check_soft_reset,
6514         .post_soft_reset = gfx_v11_0_post_soft_reset,
6515         .set_clockgating_state = gfx_v11_0_set_clockgating_state,
6516         .set_powergating_state = gfx_v11_0_set_powergating_state,
6517         .get_clockgating_state = gfx_v11_0_get_clockgating_state,
6518         .dump_ip_state = gfx_v11_ip_dump,
6519         .print_ip_state = gfx_v11_ip_print,
6520 };
6521
6522 static const struct amdgpu_ring_funcs gfx_v11_0_ring_funcs_gfx = {
6523         .type = AMDGPU_RING_TYPE_GFX,
6524         .align_mask = 0xff,
6525         .nop = PACKET3(PACKET3_NOP, 0x3FFF),
6526         .support_64bit_ptrs = true,
6527         .secure_submission_supported = true,
6528         .get_rptr = gfx_v11_0_ring_get_rptr_gfx,
6529         .get_wptr = gfx_v11_0_ring_get_wptr_gfx,
6530         .set_wptr = gfx_v11_0_ring_set_wptr_gfx,
6531         .emit_frame_size = /* totally 247 maximum if 16 IBs */
6532                 5 + /* update_spm_vmid */
6533                 5 + /* COND_EXEC */
6534                 22 + /* SET_Q_PREEMPTION_MODE */
6535                 7 + /* PIPELINE_SYNC */
6536                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
6537                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
6538                 4 + /* VM_FLUSH */
6539                 8 + /* FENCE for VM_FLUSH */
6540                 20 + /* GDS switch */
6541                 5 + /* COND_EXEC */
6542                 7 + /* HDP_flush */
6543                 4 + /* VGT_flush */
6544                 31 + /* DE_META */
6545                 3 + /* CNTX_CTRL */
6546                 5 + /* HDP_INVL */
6547                 22 + /* SET_Q_PREEMPTION_MODE */
6548                 8 + 8 + /* FENCE x2 */
6549                 8, /* gfx_v11_0_emit_mem_sync */
6550         .emit_ib_size = 4, /* gfx_v11_0_ring_emit_ib_gfx */
6551         .emit_ib = gfx_v11_0_ring_emit_ib_gfx,
6552         .emit_fence = gfx_v11_0_ring_emit_fence,
6553         .emit_pipeline_sync = gfx_v11_0_ring_emit_pipeline_sync,
6554         .emit_vm_flush = gfx_v11_0_ring_emit_vm_flush,
6555         .emit_gds_switch = gfx_v11_0_ring_emit_gds_switch,
6556         .emit_hdp_flush = gfx_v11_0_ring_emit_hdp_flush,
6557         .test_ring = gfx_v11_0_ring_test_ring,
6558         .test_ib = gfx_v11_0_ring_test_ib,
6559         .insert_nop = amdgpu_ring_insert_nop,
6560         .pad_ib = amdgpu_ring_generic_pad_ib,
6561         .emit_cntxcntl = gfx_v11_0_ring_emit_cntxcntl,
6562         .emit_gfx_shadow = gfx_v11_0_ring_emit_gfx_shadow,
6563         .init_cond_exec = gfx_v11_0_ring_emit_init_cond_exec,
6564         .preempt_ib = gfx_v11_0_ring_preempt_ib,
6565         .emit_frame_cntl = gfx_v11_0_ring_emit_frame_cntl,
6566         .emit_wreg = gfx_v11_0_ring_emit_wreg,
6567         .emit_reg_wait = gfx_v11_0_ring_emit_reg_wait,
6568         .emit_reg_write_reg_wait = gfx_v11_0_ring_emit_reg_write_reg_wait,
6569         .soft_recovery = gfx_v11_0_ring_soft_recovery,
6570         .emit_mem_sync = gfx_v11_0_emit_mem_sync,
6571 };
6572
6573 static const struct amdgpu_ring_funcs gfx_v11_0_ring_funcs_compute = {
6574         .type = AMDGPU_RING_TYPE_COMPUTE,
6575         .align_mask = 0xff,
6576         .nop = PACKET3(PACKET3_NOP, 0x3FFF),
6577         .support_64bit_ptrs = true,
6578         .get_rptr = gfx_v11_0_ring_get_rptr_compute,
6579         .get_wptr = gfx_v11_0_ring_get_wptr_compute,
6580         .set_wptr = gfx_v11_0_ring_set_wptr_compute,
6581         .emit_frame_size =
6582                 5 + /* update_spm_vmid */
6583                 20 + /* gfx_v11_0_ring_emit_gds_switch */
6584                 7 + /* gfx_v11_0_ring_emit_hdp_flush */
6585                 5 + /* hdp invalidate */
6586                 7 + /* gfx_v11_0_ring_emit_pipeline_sync */
6587                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
6588                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
6589                 2 + /* gfx_v11_0_ring_emit_vm_flush */
6590                 8 + 8 + 8 + /* gfx_v11_0_ring_emit_fence x3 for user fence, vm fence */
6591                 8, /* gfx_v11_0_emit_mem_sync */
6592         .emit_ib_size = 7, /* gfx_v11_0_ring_emit_ib_compute */
6593         .emit_ib = gfx_v11_0_ring_emit_ib_compute,
6594         .emit_fence = gfx_v11_0_ring_emit_fence,
6595         .emit_pipeline_sync = gfx_v11_0_ring_emit_pipeline_sync,
6596         .emit_vm_flush = gfx_v11_0_ring_emit_vm_flush,
6597         .emit_gds_switch = gfx_v11_0_ring_emit_gds_switch,
6598         .emit_hdp_flush = gfx_v11_0_ring_emit_hdp_flush,
6599         .test_ring = gfx_v11_0_ring_test_ring,
6600         .test_ib = gfx_v11_0_ring_test_ib,
6601         .insert_nop = amdgpu_ring_insert_nop,
6602         .pad_ib = amdgpu_ring_generic_pad_ib,
6603         .emit_wreg = gfx_v11_0_ring_emit_wreg,
6604         .emit_reg_wait = gfx_v11_0_ring_emit_reg_wait,
6605         .emit_reg_write_reg_wait = gfx_v11_0_ring_emit_reg_write_reg_wait,
6606         .emit_mem_sync = gfx_v11_0_emit_mem_sync,
6607 };
6608
6609 static const struct amdgpu_ring_funcs gfx_v11_0_ring_funcs_kiq = {
6610         .type = AMDGPU_RING_TYPE_KIQ,
6611         .align_mask = 0xff,
6612         .nop = PACKET3(PACKET3_NOP, 0x3FFF),
6613         .support_64bit_ptrs = true,
6614         .get_rptr = gfx_v11_0_ring_get_rptr_compute,
6615         .get_wptr = gfx_v11_0_ring_get_wptr_compute,
6616         .set_wptr = gfx_v11_0_ring_set_wptr_compute,
6617         .emit_frame_size =
6618                 20 + /* gfx_v11_0_ring_emit_gds_switch */
6619                 7 + /* gfx_v11_0_ring_emit_hdp_flush */
6620                 5 + /*hdp invalidate */
6621                 7 + /* gfx_v11_0_ring_emit_pipeline_sync */
6622                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
6623                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
6624                 8 + 8 + 8, /* gfx_v11_0_ring_emit_fence_kiq x3 for user fence, vm fence */
6625         .emit_ib_size = 7, /* gfx_v11_0_ring_emit_ib_compute */
6626         .emit_ib = gfx_v11_0_ring_emit_ib_compute,
6627         .emit_fence = gfx_v11_0_ring_emit_fence_kiq,
6628         .test_ring = gfx_v11_0_ring_test_ring,
6629         .test_ib = gfx_v11_0_ring_test_ib,
6630         .insert_nop = amdgpu_ring_insert_nop,
6631         .pad_ib = amdgpu_ring_generic_pad_ib,
6632         .emit_rreg = gfx_v11_0_ring_emit_rreg,
6633         .emit_wreg = gfx_v11_0_ring_emit_wreg,
6634         .emit_reg_wait = gfx_v11_0_ring_emit_reg_wait,
6635         .emit_reg_write_reg_wait = gfx_v11_0_ring_emit_reg_write_reg_wait,
6636 };
6637
6638 static void gfx_v11_0_set_ring_funcs(struct amdgpu_device *adev)
6639 {
6640         int i;
6641
6642         adev->gfx.kiq[0].ring.funcs = &gfx_v11_0_ring_funcs_kiq;
6643
6644         for (i = 0; i < adev->gfx.num_gfx_rings; i++)
6645                 adev->gfx.gfx_ring[i].funcs = &gfx_v11_0_ring_funcs_gfx;
6646
6647         for (i = 0; i < adev->gfx.num_compute_rings; i++)
6648                 adev->gfx.compute_ring[i].funcs = &gfx_v11_0_ring_funcs_compute;
6649 }
6650
6651 static const struct amdgpu_irq_src_funcs gfx_v11_0_eop_irq_funcs = {
6652         .set = gfx_v11_0_set_eop_interrupt_state,
6653         .process = gfx_v11_0_eop_irq,
6654 };
6655
6656 static const struct amdgpu_irq_src_funcs gfx_v11_0_priv_reg_irq_funcs = {
6657         .set = gfx_v11_0_set_priv_reg_fault_state,
6658         .process = gfx_v11_0_priv_reg_irq,
6659 };
6660
6661 static const struct amdgpu_irq_src_funcs gfx_v11_0_priv_inst_irq_funcs = {
6662         .set = gfx_v11_0_set_priv_inst_fault_state,
6663         .process = gfx_v11_0_priv_inst_irq,
6664 };
6665
6666 static const struct amdgpu_irq_src_funcs gfx_v11_0_rlc_gc_fed_irq_funcs = {
6667         .process = gfx_v11_0_rlc_gc_fed_irq,
6668 };
6669
6670 static void gfx_v11_0_set_irq_funcs(struct amdgpu_device *adev)
6671 {
6672         adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST;
6673         adev->gfx.eop_irq.funcs = &gfx_v11_0_eop_irq_funcs;
6674
6675         adev->gfx.priv_reg_irq.num_types = 1;
6676         adev->gfx.priv_reg_irq.funcs = &gfx_v11_0_priv_reg_irq_funcs;
6677
6678         adev->gfx.priv_inst_irq.num_types = 1;
6679         adev->gfx.priv_inst_irq.funcs = &gfx_v11_0_priv_inst_irq_funcs;
6680
6681         adev->gfx.rlc_gc_fed_irq.num_types = 1; /* 0x80 FED error */
6682         adev->gfx.rlc_gc_fed_irq.funcs = &gfx_v11_0_rlc_gc_fed_irq_funcs;
6683
6684 }
6685
6686 static void gfx_v11_0_set_imu_funcs(struct amdgpu_device *adev)
6687 {
6688         if (adev->flags & AMD_IS_APU)
6689                 adev->gfx.imu.mode = MISSION_MODE;
6690         else
6691                 adev->gfx.imu.mode = DEBUG_MODE;
6692
6693         adev->gfx.imu.funcs = &gfx_v11_0_imu_funcs;
6694 }
6695
6696 static void gfx_v11_0_set_rlc_funcs(struct amdgpu_device *adev)
6697 {
6698         adev->gfx.rlc.funcs = &gfx_v11_0_rlc_funcs;
6699 }
6700
6701 static void gfx_v11_0_set_gds_init(struct amdgpu_device *adev)
6702 {
6703         unsigned total_cu = adev->gfx.config.max_cu_per_sh *
6704                             adev->gfx.config.max_sh_per_se *
6705                             adev->gfx.config.max_shader_engines;
6706
6707         adev->gds.gds_size = 0x1000;
6708         adev->gds.gds_compute_max_wave_id = total_cu * 32 - 1;
6709         adev->gds.gws_size = 64;
6710         adev->gds.oa_size = 16;
6711 }
6712
6713 static void gfx_v11_0_set_mqd_funcs(struct amdgpu_device *adev)
6714 {
6715         /* set gfx eng mqd */
6716         adev->mqds[AMDGPU_HW_IP_GFX].mqd_size =
6717                 sizeof(struct v11_gfx_mqd);
6718         adev->mqds[AMDGPU_HW_IP_GFX].init_mqd =
6719                 gfx_v11_0_gfx_mqd_init;
6720         /* set compute eng mqd */
6721         adev->mqds[AMDGPU_HW_IP_COMPUTE].mqd_size =
6722                 sizeof(struct v11_compute_mqd);
6723         adev->mqds[AMDGPU_HW_IP_COMPUTE].init_mqd =
6724                 gfx_v11_0_compute_mqd_init;
6725 }
6726
6727 static void gfx_v11_0_set_user_wgp_inactive_bitmap_per_sh(struct amdgpu_device *adev,
6728                                                           u32 bitmap)
6729 {
6730         u32 data;
6731
6732         if (!bitmap)
6733                 return;
6734
6735         data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_WGPS__SHIFT;
6736         data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_WGPS_MASK;
6737
6738         WREG32_SOC15(GC, 0, regGC_USER_SHADER_ARRAY_CONFIG, data);
6739 }
6740
6741 static u32 gfx_v11_0_get_wgp_active_bitmap_per_sh(struct amdgpu_device *adev)
6742 {
6743         u32 data, wgp_bitmask;
6744         data = RREG32_SOC15(GC, 0, regCC_GC_SHADER_ARRAY_CONFIG);
6745         data |= RREG32_SOC15(GC, 0, regGC_USER_SHADER_ARRAY_CONFIG);
6746
6747         data &= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_WGPS_MASK;
6748         data >>= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_WGPS__SHIFT;
6749
6750         wgp_bitmask =
6751                 amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh >> 1);
6752
6753         return (~data) & wgp_bitmask;
6754 }
6755
6756 static u32 gfx_v11_0_get_cu_active_bitmap_per_sh(struct amdgpu_device *adev)
6757 {
6758         u32 wgp_idx, wgp_active_bitmap;
6759         u32 cu_bitmap_per_wgp, cu_active_bitmap;
6760
6761         wgp_active_bitmap = gfx_v11_0_get_wgp_active_bitmap_per_sh(adev);
6762         cu_active_bitmap = 0;
6763
6764         for (wgp_idx = 0; wgp_idx < 16; wgp_idx++) {
6765                 /* if there is one WGP enabled, it means 2 CUs will be enabled */
6766                 cu_bitmap_per_wgp = 3 << (2 * wgp_idx);
6767                 if (wgp_active_bitmap & (1 << wgp_idx))
6768                         cu_active_bitmap |= cu_bitmap_per_wgp;
6769         }
6770
6771         return cu_active_bitmap;
6772 }
6773
6774 static int gfx_v11_0_get_cu_info(struct amdgpu_device *adev,
6775                                  struct amdgpu_cu_info *cu_info)
6776 {
6777         int i, j, k, counter, active_cu_number = 0;
6778         u32 mask, bitmap;
6779         unsigned disable_masks[8 * 2];
6780
6781         if (!adev || !cu_info)
6782                 return -EINVAL;
6783
6784         amdgpu_gfx_parse_disable_cu(disable_masks, 8, 2);
6785
6786         mutex_lock(&adev->grbm_idx_mutex);
6787         for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
6788                 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
6789                         bitmap = i * adev->gfx.config.max_sh_per_se + j;
6790                         if (!((gfx_v11_0_get_sa_active_bitmap(adev) >> bitmap) & 1))
6791                                 continue;
6792                         mask = 1;
6793                         counter = 0;
6794                         gfx_v11_0_select_se_sh(adev, i, j, 0xffffffff, 0);
6795                         if (i < 8 && j < 2)
6796                                 gfx_v11_0_set_user_wgp_inactive_bitmap_per_sh(
6797                                         adev, disable_masks[i * 2 + j]);
6798                         bitmap = gfx_v11_0_get_cu_active_bitmap_per_sh(adev);
6799
6800                         /**
6801                          * GFX11 could support more than 4 SEs, while the bitmap
6802                          * in cu_info struct is 4x4 and ioctl interface struct
6803                          * drm_amdgpu_info_device should keep stable.
6804                          * So we use last two columns of bitmap to store cu mask for
6805                          * SEs 4 to 7, the layout of the bitmap is as below:
6806                          *    SE0: {SH0,SH1} --> {bitmap[0][0], bitmap[0][1]}
6807                          *    SE1: {SH0,SH1} --> {bitmap[1][0], bitmap[1][1]}
6808                          *    SE2: {SH0,SH1} --> {bitmap[2][0], bitmap[2][1]}
6809                          *    SE3: {SH0,SH1} --> {bitmap[3][0], bitmap[3][1]}
6810                          *    SE4: {SH0,SH1} --> {bitmap[0][2], bitmap[0][3]}
6811                          *    SE5: {SH0,SH1} --> {bitmap[1][2], bitmap[1][3]}
6812                          *    SE6: {SH0,SH1} --> {bitmap[2][2], bitmap[2][3]}
6813                          *    SE7: {SH0,SH1} --> {bitmap[3][2], bitmap[3][3]}
6814                          */
6815                         cu_info->bitmap[0][i % 4][j + (i / 4) * 2] = bitmap;
6816
6817                         for (k = 0; k < adev->gfx.config.max_cu_per_sh; k++) {
6818                                 if (bitmap & mask)
6819                                         counter++;
6820
6821                                 mask <<= 1;
6822                         }
6823                         active_cu_number += counter;
6824                 }
6825         }
6826         gfx_v11_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
6827         mutex_unlock(&adev->grbm_idx_mutex);
6828
6829         cu_info->number = active_cu_number;
6830         cu_info->simd_per_cu = NUM_SIMD_PER_CU;
6831
6832         return 0;
6833 }
6834
6835 const struct amdgpu_ip_block_version gfx_v11_0_ip_block =
6836 {
6837         .type = AMD_IP_BLOCK_TYPE_GFX,
6838         .major = 11,
6839         .minor = 0,
6840         .rev = 0,
6841         .funcs = &gfx_v11_0_ip_funcs,
6842 };