drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c

   1 /*
   2  * Copyright 2016 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23 #include <linux/kernel.h>
  24 #include <linux/firmware.h>
  25 #include <drm/drmP.h>
  26 #include "amdgpu.h"
  27 #include "amdgpu_gfx.h"
  28 #include "soc15.h"
  29 #include "soc15d.h"
  30
  31 #include "gc/gc_9_0_offset.h"
  32 #include "gc/gc_9_0_sh_mask.h"
  33 #include "vega10_enum.h"
  34 #include "hdp/hdp_4_0_offset.h"
  35
  36 #include "soc15_common.h"
  37 #include "clearstate_gfx9.h"
  38 #include "v9_structs.h"
  39
  40 #define GFX9_NUM_GFX_RINGS     1
  41 #define GFX9_MEC_HPD_SIZE 2048
  42 #define RLCG_UCODE_LOADING_START_ADDRESS 0x00002000L
  43 #define RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET 0x00000000L
  44
  45 #define mmPWR_MISC_CNTL_STATUS                                  0x0183
  46 #define mmPWR_MISC_CNTL_STATUS_BASE_IDX                         0
  47 #define PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN__SHIFT        0x0
  48 #define PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS__SHIFT          0x1
  49 #define PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK          0x00000001L
  50 #define PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS_MASK            0x00000006L
  51
  52 MODULE_FIRMWARE("amdgpu/vega10_ce.bin");
  53 MODULE_FIRMWARE("amdgpu/vega10_pfp.bin");
  54 MODULE_FIRMWARE("amdgpu/vega10_me.bin");
  55 MODULE_FIRMWARE("amdgpu/vega10_mec.bin");
  56 MODULE_FIRMWARE("amdgpu/vega10_mec2.bin");
  57 MODULE_FIRMWARE("amdgpu/vega10_rlc.bin");
  58
  59 MODULE_FIRMWARE("amdgpu/vega12_ce.bin");
  60 MODULE_FIRMWARE("amdgpu/vega12_pfp.bin");
  61 MODULE_FIRMWARE("amdgpu/vega12_me.bin");
  62 MODULE_FIRMWARE("amdgpu/vega12_mec.bin");
  63 MODULE_FIRMWARE("amdgpu/vega12_mec2.bin");
  64 MODULE_FIRMWARE("amdgpu/vega12_rlc.bin");
  65
  66 MODULE_FIRMWARE("amdgpu/raven_ce.bin");
  67 MODULE_FIRMWARE("amdgpu/raven_pfp.bin");
  68 MODULE_FIRMWARE("amdgpu/raven_me.bin");
  69 MODULE_FIRMWARE("amdgpu/raven_mec.bin");
  70 MODULE_FIRMWARE("amdgpu/raven_mec2.bin");
  71 MODULE_FIRMWARE("amdgpu/raven_rlc.bin");
  72
  73 static const struct soc15_reg_golden golden_settings_gc_9_0[] =
  74 {
  75         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080),
  76         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080),
  77         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080),
  78         SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420),
  79         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000),
  80         SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080),
  81         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024),
  82         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
  83         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
  84         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080),
  85         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080),
  86         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080),
  87         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080),
  88         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080),
  89         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000),
  90         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x01000107),
  91         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000),
  92         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
  93         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68),
  94         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197),
  95         SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000),
  96         SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff),
  97         SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080)
  98 };
  99
 100 static const struct soc15_reg_golden golden_settings_gc_9_0_vg10[] =
 101 {
 102         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0x0000f000, 0x00012107),
 103         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000),
 104         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042),
 105         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x2a114042),
 106         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0x00008000, 0x00048000),
 107         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000),
 108         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x00001800, 0x00000800)
 109 };
 110
 111 static const struct soc15_reg_golden golden_settings_gc_9_1[] =
 112 {
 113         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
 114         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080),
 115         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080),
 116         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080),
 117         SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420),
 118         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000),
 119         SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080),
 120         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024),
 121         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
 122         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
 123         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080),
 124         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080),
 125         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080),
 126         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080),
 127         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080),
 128         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
 129         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
 130         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120),
 131         SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000),
 132         SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000000ff),
 133         SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080)
 134 };
 135
 136 static const struct soc15_reg_golden golden_settings_gc_9_1_rv1[] =
 137 {
 138         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000),
 139         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24000042),
 140         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24000042),
 141         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04048000),
 142         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_MODE_CNTL_1, 0x06000000, 0x06000000),
 143         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000),
 144         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x00000800)
 145 };
 146
 147 static const struct soc15_reg_golden golden_settings_gc_9_x_common[] =
 148 {
 149         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_INDEX, 0xffffffff, 0x00000000),
 150         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_DATA, 0xffffffff, 0x2544c382)
 151 };
 152
 153 static const struct soc15_reg_golden golden_settings_gc_9_2_1[] =
 154 {
 155         SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420),
 156         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000),
 157         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024),
 158         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
 159         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
 160         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000),
 161         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800),
 162         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800),
 163         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x0000ff87),
 164         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x0000ff8f),
 165         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000),
 166         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
 167         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68),
 168         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197),
 169         SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000),
 170         SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff)
 171 };
 172
 173 static const struct soc15_reg_golden golden_settings_gc_9_2_1_vg12[] =
 174 {
 175         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0x00000080, 0x04000080),
 176         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
 177         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000),
 178         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24104041),
 179         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24104041),
 180         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000),
 181         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff03ff, 0x01000107),
 182         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
 183         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x76325410),
 184         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000)
 185 };
 186
 187 static const u32 GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[] =
 188 {
 189         mmRLC_SRM_INDEX_CNTL_ADDR_0 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 190         mmRLC_SRM_INDEX_CNTL_ADDR_1 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 191         mmRLC_SRM_INDEX_CNTL_ADDR_2 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 192         mmRLC_SRM_INDEX_CNTL_ADDR_3 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 193         mmRLC_SRM_INDEX_CNTL_ADDR_4 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 194         mmRLC_SRM_INDEX_CNTL_ADDR_5 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 195         mmRLC_SRM_INDEX_CNTL_ADDR_6 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 196         mmRLC_SRM_INDEX_CNTL_ADDR_7 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 197 };
 198
 199 static const u32 GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[] =
 200 {
 201         mmRLC_SRM_INDEX_CNTL_DATA_0 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 202         mmRLC_SRM_INDEX_CNTL_DATA_1 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 203         mmRLC_SRM_INDEX_CNTL_DATA_2 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 204         mmRLC_SRM_INDEX_CNTL_DATA_3 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 205         mmRLC_SRM_INDEX_CNTL_DATA_4 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 206         mmRLC_SRM_INDEX_CNTL_DATA_5 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 207         mmRLC_SRM_INDEX_CNTL_DATA_6 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 208         mmRLC_SRM_INDEX_CNTL_DATA_7 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 209 };
 210
 211 #define VEGA10_GB_ADDR_CONFIG_GOLDEN 0x2a114042
 212 #define VEGA12_GB_ADDR_CONFIG_GOLDEN 0x24104041
 213 #define RAVEN_GB_ADDR_CONFIG_GOLDEN 0x24000042
 214
 215 static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev);
 216 static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev);
 217 static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev);
 218 static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev);
 219 static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev,
 220                                  struct amdgpu_cu_info *cu_info);
 221 static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev);
 222 static void gfx_v9_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num, u32 instance);
 223 static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring);
 224
 225 static void gfx_v9_0_init_golden_registers(struct amdgpu_device *adev)
 226 {
 227         switch (adev->asic_type) {
 228         case CHIP_VEGA10:
 229                 soc15_program_register_sequence(adev,
 230                                                  golden_settings_gc_9_0,
 231                                                  ARRAY_SIZE(golden_settings_gc_9_0));
 232                 soc15_program_register_sequence(adev,
 233                                                  golden_settings_gc_9_0_vg10,
 234                                                  ARRAY_SIZE(golden_settings_gc_9_0_vg10));
 235                 break;
 236         case CHIP_VEGA12:
 237                 soc15_program_register_sequence(adev,
 238                                                 golden_settings_gc_9_2_1,
 239                                                 ARRAY_SIZE(golden_settings_gc_9_2_1));
 240                 soc15_program_register_sequence(adev,
 241                                                 golden_settings_gc_9_2_1_vg12,
 242                                                 ARRAY_SIZE(golden_settings_gc_9_2_1_vg12));
 243                 break;
 244         case CHIP_RAVEN:
 245                 soc15_program_register_sequence(adev,
 246                                                  golden_settings_gc_9_1,
 247                                                  ARRAY_SIZE(golden_settings_gc_9_1));
 248                 soc15_program_register_sequence(adev,
 249                                                  golden_settings_gc_9_1_rv1,
 250                                                  ARRAY_SIZE(golden_settings_gc_9_1_rv1));
 251                 break;
 252         default:
 253                 break;
 254         }
 255
 256         soc15_program_register_sequence(adev, golden_settings_gc_9_x_common,
 257                                         (const u32)ARRAY_SIZE(golden_settings_gc_9_x_common));
 258 }
 259
 260 static void gfx_v9_0_scratch_init(struct amdgpu_device *adev)
 261 {
 262         adev->gfx.scratch.num_reg = 8;
 263         adev->gfx.scratch.reg_base = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG0);
 264         adev->gfx.scratch.free_mask = (1u << adev->gfx.scratch.num_reg) - 1;
 265 }
 266
 267 static void gfx_v9_0_write_data_to_reg(struct amdgpu_ring *ring, int eng_sel,
 268                                        bool wc, uint32_t reg, uint32_t val)
 269 {
 270         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
 271         amdgpu_ring_write(ring, WRITE_DATA_ENGINE_SEL(eng_sel) |
 272                                 WRITE_DATA_DST_SEL(0) |
 273                                 (wc ? WR_CONFIRM : 0));
 274         amdgpu_ring_write(ring, reg);
 275         amdgpu_ring_write(ring, 0);
 276         amdgpu_ring_write(ring, val);
 277 }
 278
 279 static void gfx_v9_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel,
 280                                   int mem_space, int opt, uint32_t addr0,
 281                                   uint32_t addr1, uint32_t ref, uint32_t mask,
 282                                   uint32_t inv)
 283 {
 284         amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
 285         amdgpu_ring_write(ring,
 286                                  /* memory (1) or register (0) */
 287                                  (WAIT_REG_MEM_MEM_SPACE(mem_space) |
 288                                  WAIT_REG_MEM_OPERATION(opt) | /* wait */
 289                                  WAIT_REG_MEM_FUNCTION(3) |  /* equal */
 290                                  WAIT_REG_MEM_ENGINE(eng_sel)));
 291
 292         if (mem_space)
 293                 BUG_ON(addr0 & 0x3); /* Dword align */
 294         amdgpu_ring_write(ring, addr0);
 295         amdgpu_ring_write(ring, addr1);
 296         amdgpu_ring_write(ring, ref);
 297         amdgpu_ring_write(ring, mask);
 298         amdgpu_ring_write(ring, inv); /* poll interval */
 299 }
 300
 301 static int gfx_v9_0_ring_test_ring(struct amdgpu_ring *ring)
 302 {
 303         struct amdgpu_device *adev = ring->adev;
 304         uint32_t scratch;
 305         uint32_t tmp = 0;
 306         unsigned i;
 307         int r;
 308
 309         r = amdgpu_gfx_scratch_get(adev, &scratch);
 310         if (r) {
 311                 DRM_ERROR("amdgpu: cp failed to get scratch reg (%d).\n", r);
 312                 return r;
 313         }
 314         WREG32(scratch, 0xCAFEDEAD);
 315         r = amdgpu_ring_alloc(ring, 3);
 316         if (r) {
 317                 DRM_ERROR("amdgpu: cp failed to lock ring %d (%d).\n",
 318                           ring->idx, r);
 319                 amdgpu_gfx_scratch_free(adev, scratch);
 320                 return r;
 321         }
 322         amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
 323         amdgpu_ring_write(ring, (scratch - PACKET3_SET_UCONFIG_REG_START));
 324         amdgpu_ring_write(ring, 0xDEADBEEF);
 325         amdgpu_ring_commit(ring);
 326
 327         for (i = 0; i < adev->usec_timeout; i++) {
 328                 tmp = RREG32(scratch);
 329                 if (tmp == 0xDEADBEEF)
 330                         break;
 331                 DRM_UDELAY(1);
 332         }
 333         if (i < adev->usec_timeout) {
 334                 DRM_DEBUG("ring test on %d succeeded in %d usecs\n",
 335                          ring->idx, i);
 336         } else {
 337                 DRM_ERROR("amdgpu: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
 338                           ring->idx, scratch, tmp);
 339                 r = -EINVAL;
 340         }
 341         amdgpu_gfx_scratch_free(adev, scratch);
 342         return r;
 343 }
 344
 345 static int gfx_v9_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 346 {
 347         struct amdgpu_device *adev = ring->adev;
 348         struct amdgpu_ib ib;
 349         struct dma_fence *f = NULL;
 350
 351         unsigned index;
 352         uint64_t gpu_addr;
 353         uint32_t tmp;
 354         long r;
 355
 356         r = amdgpu_device_wb_get(adev, &index);
 357         if (r) {
 358                 dev_err(adev->dev, "(%ld) failed to allocate wb slot\n", r);
 359                 return r;
 360         }
 361
 362         gpu_addr = adev->wb.gpu_addr + (index * 4);
 363         adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD);
 364         memset(&ib, 0, sizeof(ib));
 365         r = amdgpu_ib_get(adev, NULL, 16, &ib);
 366         if (r) {
 367                 DRM_ERROR("amdgpu: failed to get ib (%ld).\n", r);
 368                 goto err1;
 369         }
 370         ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3);
 371         ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM;
 372         ib.ptr[2] = lower_32_bits(gpu_addr);
 373         ib.ptr[3] = upper_32_bits(gpu_addr);
 374         ib.ptr[4] = 0xDEADBEEF;
 375         ib.length_dw = 5;
 376
 377         r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
 378         if (r)
 379                 goto err2;
 380
 381         r = dma_fence_wait_timeout(f, false, timeout);
 382         if (r == 0) {
 383                         DRM_ERROR("amdgpu: IB test timed out.\n");
 384                         r = -ETIMEDOUT;
 385                         goto err2;
 386         } else if (r < 0) {
 387                         DRM_ERROR("amdgpu: fence wait failed (%ld).\n", r);
 388                         goto err2;
 389         }
 390
 391         tmp = adev->wb.wb[index];
 392         if (tmp == 0xDEADBEEF) {
 393                         DRM_DEBUG("ib test on ring %d succeeded\n", ring->idx);
 394                         r = 0;
 395         } else {
 396                         DRM_ERROR("ib test on ring %d failed\n", ring->idx);
 397                         r = -EINVAL;
 398         }
 399
 400 err2:
 401         amdgpu_ib_free(adev, &ib, NULL);
 402         dma_fence_put(f);
 403 err1:
 404         amdgpu_device_wb_free(adev, index);
 405         return r;
 406 }
 407
 408
 409 static void gfx_v9_0_free_microcode(struct amdgpu_device *adev)
 410 {
 411         release_firmware(adev->gfx.pfp_fw);
 412         adev->gfx.pfp_fw = NULL;
 413         release_firmware(adev->gfx.me_fw);
 414         adev->gfx.me_fw = NULL;
 415         release_firmware(adev->gfx.ce_fw);
 416         adev->gfx.ce_fw = NULL;
 417         release_firmware(adev->gfx.rlc_fw);
 418         adev->gfx.rlc_fw = NULL;
 419         release_firmware(adev->gfx.mec_fw);
 420         adev->gfx.mec_fw = NULL;
 421         release_firmware(adev->gfx.mec2_fw);
 422         adev->gfx.mec2_fw = NULL;
 423
 424         kfree(adev->gfx.rlc.register_list_format);
 425 }
 426
 427 static void gfx_v9_0_init_rlc_ext_microcode(struct amdgpu_device *adev)
 428 {
 429         const struct rlc_firmware_header_v2_1 *rlc_hdr;
 430
 431         rlc_hdr = (const struct rlc_firmware_header_v2_1 *)adev->gfx.rlc_fw->data;
 432         adev->gfx.rlc_srlc_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_cntl_ucode_ver);
 433         adev->gfx.rlc_srlc_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_cntl_feature_ver);
 434         adev->gfx.rlc.save_restore_list_cntl_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_cntl_size_bytes);
 435         adev->gfx.rlc.save_restore_list_cntl = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_cntl_offset_bytes);
 436         adev->gfx.rlc_srlg_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_gpm_ucode_ver);
 437         adev->gfx.rlc_srlg_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_gpm_feature_ver);
 438         adev->gfx.rlc.save_restore_list_gpm_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_gpm_size_bytes);
 439         adev->gfx.rlc.save_restore_list_gpm = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_gpm_offset_bytes);
 440         adev->gfx.rlc_srls_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_srm_ucode_ver);
 441         adev->gfx.rlc_srls_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_srm_feature_ver);
 442         adev->gfx.rlc.save_restore_list_srm_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_srm_size_bytes);
 443         adev->gfx.rlc.save_restore_list_srm = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_srm_offset_bytes);
 444         adev->gfx.rlc.reg_list_format_direct_reg_list_length =
 445                         le32_to_cpu(rlc_hdr->reg_list_format_direct_reg_list_length);
 446 }
 447
 448 static int gfx_v9_0_init_microcode(struct amdgpu_device *adev)
 449 {
 450         const char *chip_name;
 451         char fw_name[30];
 452         int err;
 453         struct amdgpu_firmware_info *info = NULL;
 454         const struct common_firmware_header *header = NULL;
 455         const struct gfx_firmware_header_v1_0 *cp_hdr;
 456         const struct rlc_firmware_header_v2_0 *rlc_hdr;
 457         unsigned int *tmp = NULL;
 458         unsigned int i = 0;
 459         uint16_t version_major;
 460         uint16_t version_minor;
 461
 462         DRM_DEBUG("\n");
 463
 464         switch (adev->asic_type) {
 465         case CHIP_VEGA10:
 466                 chip_name = "vega10";
 467                 break;
 468         case CHIP_VEGA12:
 469                 chip_name = "vega12";
 470                 break;
 471         case CHIP_RAVEN:
 472                 chip_name = "raven";
 473                 break;
 474         default:
 475                 BUG();
 476         }
 477
 478         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_pfp.bin", chip_name);
 479         err = request_firmware(&adev->gfx.pfp_fw, fw_name, adev->dev);
 480         if (err)
 481                 goto out;
 482         err = amdgpu_ucode_validate(adev->gfx.pfp_fw);
 483         if (err)
 484                 goto out;
 485         cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.pfp_fw->data;
 486         adev->gfx.pfp_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
 487         adev->gfx.pfp_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
 488
 489         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_me.bin", chip_name);
 490         err = request_firmware(&adev->gfx.me_fw, fw_name, adev->dev);
 491         if (err)
 492                 goto out;
 493         err = amdgpu_ucode_validate(adev->gfx.me_fw);
 494         if (err)
 495                 goto out;
 496         cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.me_fw->data;
 497         adev->gfx.me_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
 498         adev->gfx.me_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
 499
 500         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_ce.bin", chip_name);
 501         err = request_firmware(&adev->gfx.ce_fw, fw_name, adev->dev);
 502         if (err)
 503                 goto out;
 504         err = amdgpu_ucode_validate(adev->gfx.ce_fw);
 505         if (err)
 506                 goto out;
 507         cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.ce_fw->data;
 508         adev->gfx.ce_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
 509         adev->gfx.ce_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
 510
 511         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_rlc.bin", chip_name);
 512         err = request_firmware(&adev->gfx.rlc_fw, fw_name, adev->dev);
 513         if (err)
 514                 goto out;
 515         err = amdgpu_ucode_validate(adev->gfx.rlc_fw);
 516         rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
 517
 518         version_major = le16_to_cpu(rlc_hdr->header.header_version_major);
 519         version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor);
 520         if (version_major == 2 && version_minor == 1)
 521                 adev->gfx.rlc.is_rlc_v2_1 = true;
 522
 523         adev->gfx.rlc_fw_version = le32_to_cpu(rlc_hdr->header.ucode_version);
 524         adev->gfx.rlc_feature_version = le32_to_cpu(rlc_hdr->ucode_feature_version);
 525         adev->gfx.rlc.save_and_restore_offset =
 526                         le32_to_cpu(rlc_hdr->save_and_restore_offset);
 527         adev->gfx.rlc.clear_state_descriptor_offset =
 528                         le32_to_cpu(rlc_hdr->clear_state_descriptor_offset);
 529         adev->gfx.rlc.avail_scratch_ram_locations =
 530                         le32_to_cpu(rlc_hdr->avail_scratch_ram_locations);
 531         adev->gfx.rlc.reg_restore_list_size =
 532                         le32_to_cpu(rlc_hdr->reg_restore_list_size);
 533         adev->gfx.rlc.reg_list_format_start =
 534                         le32_to_cpu(rlc_hdr->reg_list_format_start);
 535         adev->gfx.rlc.reg_list_format_separate_start =
 536                         le32_to_cpu(rlc_hdr->reg_list_format_separate_start);
 537         adev->gfx.rlc.starting_offsets_start =
 538                         le32_to_cpu(rlc_hdr->starting_offsets_start);
 539         adev->gfx.rlc.reg_list_format_size_bytes =
 540                         le32_to_cpu(rlc_hdr->reg_list_format_size_bytes);
 541         adev->gfx.rlc.reg_list_size_bytes =
 542                         le32_to_cpu(rlc_hdr->reg_list_size_bytes);
 543         adev->gfx.rlc.register_list_format =
 544                         kmalloc(adev->gfx.rlc.reg_list_format_size_bytes +
 545                                 adev->gfx.rlc.reg_list_size_bytes, GFP_KERNEL);
 546         if (!adev->gfx.rlc.register_list_format) {
 547                 err = -ENOMEM;
 548                 goto out;
 549         }
 550
 551         tmp = (unsigned int *)((uintptr_t)rlc_hdr +
 552                         le32_to_cpu(rlc_hdr->reg_list_format_array_offset_bytes));
 553         for (i = 0 ; i < (rlc_hdr->reg_list_format_size_bytes >> 2); i++)
 554                 adev->gfx.rlc.register_list_format[i] = le32_to_cpu(tmp[i]);
 555
 556         adev->gfx.rlc.register_restore = adev->gfx.rlc.register_list_format + i;
 557
 558         tmp = (unsigned int *)((uintptr_t)rlc_hdr +
 559                         le32_to_cpu(rlc_hdr->reg_list_array_offset_bytes));
 560         for (i = 0 ; i < (rlc_hdr->reg_list_size_bytes >> 2); i++)
 561                 adev->gfx.rlc.register_restore[i] = le32_to_cpu(tmp[i]);
 562
 563         if (adev->gfx.rlc.is_rlc_v2_1)
 564                 gfx_v9_0_init_rlc_ext_microcode(adev);
 565
 566         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec.bin", chip_name);
 567         err = request_firmware(&adev->gfx.mec_fw, fw_name, adev->dev);
 568         if (err)
 569                 goto out;
 570         err = amdgpu_ucode_validate(adev->gfx.mec_fw);
 571         if (err)
 572                 goto out;
 573         cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
 574         adev->gfx.mec_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
 575         adev->gfx.mec_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
 576
 577
 578         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec2.bin", chip_name);
 579         err = request_firmware(&adev->gfx.mec2_fw, fw_name, adev->dev);
 580         if (!err) {
 581                 err = amdgpu_ucode_validate(adev->gfx.mec2_fw);
 582                 if (err)
 583                         goto out;
 584                 cp_hdr = (const struct gfx_firmware_header_v1_0 *)
 585                 adev->gfx.mec2_fw->data;
 586                 adev->gfx.mec2_fw_version =
 587                 le32_to_cpu(cp_hdr->header.ucode_version);
 588                 adev->gfx.mec2_feature_version =
 589                 le32_to_cpu(cp_hdr->ucode_feature_version);
 590         } else {
 591                 err = 0;
 592                 adev->gfx.mec2_fw = NULL;
 593         }
 594
 595         if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
 596                 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_PFP];
 597                 info->ucode_id = AMDGPU_UCODE_ID_CP_PFP;
 598                 info->fw = adev->gfx.pfp_fw;
 599                 header = (const struct common_firmware_header *)info->fw->data;
 600                 adev->firmware.fw_size +=
 601                         ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
 602
 603                 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_ME];
 604                 info->ucode_id = AMDGPU_UCODE_ID_CP_ME;
 605                 info->fw = adev->gfx.me_fw;
 606                 header = (const struct common_firmware_header *)info->fw->data;
 607                 adev->firmware.fw_size +=
 608                         ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
 609
 610                 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_CE];
 611                 info->ucode_id = AMDGPU_UCODE_ID_CP_CE;
 612                 info->fw = adev->gfx.ce_fw;
 613                 header = (const struct common_firmware_header *)info->fw->data;
 614                 adev->firmware.fw_size +=
 615                         ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
 616
 617                 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_G];
 618                 info->ucode_id = AMDGPU_UCODE_ID_RLC_G;
 619                 info->fw = adev->gfx.rlc_fw;
 620                 header = (const struct common_firmware_header *)info->fw->data;
 621                 adev->firmware.fw_size +=
 622                         ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
 623
 624                 if (adev->gfx.rlc.is_rlc_v2_1) {
 625                         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL];
 626                         info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL;
 627                         info->fw = adev->gfx.rlc_fw;
 628                         adev->firmware.fw_size +=
 629                                 ALIGN(adev->gfx.rlc.save_restore_list_cntl_size_bytes, PAGE_SIZE);
 630
 631                         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM];
 632                         info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM;
 633                         info->fw = adev->gfx.rlc_fw;
 634                         adev->firmware.fw_size +=
 635                                 ALIGN(adev->gfx.rlc.save_restore_list_gpm_size_bytes, PAGE_SIZE);
 636
 637                         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM];
 638                         info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM;
 639                         info->fw = adev->gfx.rlc_fw;
 640                         adev->firmware.fw_size +=
 641                                 ALIGN(adev->gfx.rlc.save_restore_list_srm_size_bytes, PAGE_SIZE);
 642                 }
 643
 644                 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1];
 645                 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1;
 646                 info->fw = adev->gfx.mec_fw;
 647                 header = (const struct common_firmware_header *)info->fw->data;
 648                 cp_hdr = (const struct gfx_firmware_header_v1_0 *)info->fw->data;
 649                 adev->firmware.fw_size +=
 650                         ALIGN(le32_to_cpu(header->ucode_size_bytes) - le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE);
 651
 652                 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1_JT];
 653                 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1_JT;
 654                 info->fw = adev->gfx.mec_fw;
 655                 adev->firmware.fw_size +=
 656                         ALIGN(le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE);
 657
 658                 if (adev->gfx.mec2_fw) {
 659                         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC2];
 660                         info->ucode_id = AMDGPU_UCODE_ID_CP_MEC2;
 661                         info->fw = adev->gfx.mec2_fw;
 662                         header = (const struct common_firmware_header *)info->fw->data;
 663                         cp_hdr = (const struct gfx_firmware_header_v1_0 *)info->fw->data;
 664                         adev->firmware.fw_size +=
 665                                 ALIGN(le32_to_cpu(header->ucode_size_bytes) - le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE);
 666                         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC2_JT];
 667                         info->ucode_id = AMDGPU_UCODE_ID_CP_MEC2_JT;
 668                         info->fw = adev->gfx.mec2_fw;
 669                         adev->firmware.fw_size +=
 670                                 ALIGN(le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE);
 671                 }
 672
 673         }
 674
 675 out:
 676         if (err) {
 677                 dev_err(adev->dev,
 678                         "gfx9: Failed to load firmware \"%s\"\n",
 679                         fw_name);
 680                 release_firmware(adev->gfx.pfp_fw);
 681                 adev->gfx.pfp_fw = NULL;
 682                 release_firmware(adev->gfx.me_fw);
 683                 adev->gfx.me_fw = NULL;
 684                 release_firmware(adev->gfx.ce_fw);
 685                 adev->gfx.ce_fw = NULL;
 686                 release_firmware(adev->gfx.rlc_fw);
 687                 adev->gfx.rlc_fw = NULL;
 688                 release_firmware(adev->gfx.mec_fw);
 689                 adev->gfx.mec_fw = NULL;
 690                 release_firmware(adev->gfx.mec2_fw);
 691                 adev->gfx.mec2_fw = NULL;
 692         }
 693         return err;
 694 }
 695
 696 static u32 gfx_v9_0_get_csb_size(struct amdgpu_device *adev)
 697 {
 698         u32 count = 0;
 699         const struct cs_section_def *sect = NULL;
 700         const struct cs_extent_def *ext = NULL;
 701
 702         /* begin clear state */
 703         count += 2;
 704         /* context control state */
 705         count += 3;
 706
 707         for (sect = gfx9_cs_data; sect->section != NULL; ++sect) {
 708                 for (ext = sect->section; ext->extent != NULL; ++ext) {
 709                         if (sect->id == SECT_CONTEXT)
 710                                 count += 2 + ext->reg_count;
 711                         else
 712                                 return 0;
 713                 }
 714         }
 715
 716         /* end clear state */
 717         count += 2;
 718         /* clear state */
 719         count += 2;
 720
 721         return count;
 722 }
 723
 724 static void gfx_v9_0_get_csb_buffer(struct amdgpu_device *adev,
 725                                     volatile u32 *buffer)
 726 {
 727         u32 count = 0, i;
 728         const struct cs_section_def *sect = NULL;
 729         const struct cs_extent_def *ext = NULL;
 730
 731         if (adev->gfx.rlc.cs_data == NULL)
 732                 return;
 733         if (buffer == NULL)
 734                 return;
 735
 736         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
 737         buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
 738
 739         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
 740         buffer[count++] = cpu_to_le32(0x80000000);
 741         buffer[count++] = cpu_to_le32(0x80000000);
 742
 743         for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) {
 744                 for (ext = sect->section; ext->extent != NULL; ++ext) {
 745                         if (sect->id == SECT_CONTEXT) {
 746                                 buffer[count++] =
 747                                         cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
 748                                 buffer[count++] = cpu_to_le32(ext->reg_index -
 749                                                 PACKET3_SET_CONTEXT_REG_START);
 750                                 for (i = 0; i < ext->reg_count; i++)
 751                                         buffer[count++] = cpu_to_le32(ext->extent[i]);
 752                         } else {
 753                                 return;
 754                         }
 755                 }
 756         }
 757
 758         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
 759         buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
 760
 761         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
 762         buffer[count++] = cpu_to_le32(0);
 763 }
 764
 765 static void gfx_v9_0_init_lbpw(struct amdgpu_device *adev)
 766 {
 767         uint32_t data;
 768
 769         /* set mmRLC_LB_THR_CONFIG_1/2/3/4 */
 770         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_1, 0x0000007F);
 771         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_2, 0x0333A5A7);
 772         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_3, 0x00000077);
 773         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_4, (0x30 | 0x40 << 8 | 0x02FA << 16));
 774
 775         /* set mmRLC_LB_CNTR_INIT = 0x0000_0000 */
 776         WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_INIT, 0x00000000);
 777
 778         /* set mmRLC_LB_CNTR_MAX = 0x0000_0500 */
 779         WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_MAX, 0x00000500);
 780
 781         mutex_lock(&adev->grbm_idx_mutex);
 782         /* set mmRLC_LB_INIT_CU_MASK thru broadcast mode to enable all SE/SH*/
 783         gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
 784         WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff);
 785
 786         /* set mmRLC_LB_PARAMS = 0x003F_1006 */
 787         data = REG_SET_FIELD(0, RLC_LB_PARAMS, FIFO_SAMPLES, 0x0003);
 788         data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLES, 0x0010);
 789         data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLE_INTERVAL, 0x033F);
 790         WREG32_SOC15(GC, 0, mmRLC_LB_PARAMS, data);
 791
 792         /* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */
 793         data = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7);
 794         data &= 0x0000FFFF;
 795         data |= 0x00C00000;
 796         WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7, data);
 797
 798         /* set RLC_LB_ALWAYS_ACTIVE_CU_MASK = 0xFFF */
 799         WREG32_SOC15(GC, 0, mmRLC_LB_ALWAYS_ACTIVE_CU_MASK, 0xFFF);
 800
 801         /* set RLC_LB_CNTL = 0x8000_0095, 31 bit is reserved,
 802          * but used for RLC_LB_CNTL configuration */
 803         data = RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK;
 804         data |= REG_SET_FIELD(data, RLC_LB_CNTL, CU_MASK_USED_OFF_HYST, 0x09);
 805         data |= REG_SET_FIELD(data, RLC_LB_CNTL, RESERVED, 0x80000);
 806         WREG32_SOC15(GC, 0, mmRLC_LB_CNTL, data);
 807         mutex_unlock(&adev->grbm_idx_mutex);
 808 }
 809
 810 static void gfx_v9_0_enable_lbpw(struct amdgpu_device *adev, bool enable)
 811 {
 812         WREG32_FIELD15(GC, 0, RLC_LB_CNTL, LOAD_BALANCE_ENABLE, enable ? 1 : 0);
 813 }
 814
 815 static void rv_init_cp_jump_table(struct amdgpu_device *adev)
 816 {
 817         const __le32 *fw_data;
 818         volatile u32 *dst_ptr;
 819         int me, i, max_me = 5;
 820         u32 bo_offset = 0;
 821         u32 table_offset, table_size;
 822
 823         /* write the cp table buffer */
 824         dst_ptr = adev->gfx.rlc.cp_table_ptr;
 825         for (me = 0; me < max_me; me++) {
 826                 if (me == 0) {
 827                         const struct gfx_firmware_header_v1_0 *hdr =
 828                                 (const struct gfx_firmware_header_v1_0 *)adev->gfx.ce_fw->data;
 829                         fw_data = (const __le32 *)
 830                                 (adev->gfx.ce_fw->data +
 831                                  le32_to_cpu(hdr->header.ucode_array_offset_bytes));
 832                         table_offset = le32_to_cpu(hdr->jt_offset);
 833                         table_size = le32_to_cpu(hdr->jt_size);
 834                 } else if (me == 1) {
 835                         const struct gfx_firmware_header_v1_0 *hdr =
 836                                 (const struct gfx_firmware_header_v1_0 *)adev->gfx.pfp_fw->data;
 837                         fw_data = (const __le32 *)
 838                                 (adev->gfx.pfp_fw->data +
 839                                  le32_to_cpu(hdr->header.ucode_array_offset_bytes));
 840                         table_offset = le32_to_cpu(hdr->jt_offset);
 841                         table_size = le32_to_cpu(hdr->jt_size);
 842                 } else if (me == 2) {
 843                         const struct gfx_firmware_header_v1_0 *hdr =
 844                                 (const struct gfx_firmware_header_v1_0 *)adev->gfx.me_fw->data;
 845                         fw_data = (const __le32 *)
 846                                 (adev->gfx.me_fw->data +
 847                                  le32_to_cpu(hdr->header.ucode_array_offset_bytes));
 848                         table_offset = le32_to_cpu(hdr->jt_offset);
 849                         table_size = le32_to_cpu(hdr->jt_size);
 850                 } else if (me == 3) {
 851                         const struct gfx_firmware_header_v1_0 *hdr =
 852                                 (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
 853                         fw_data = (const __le32 *)
 854                                 (adev->gfx.mec_fw->data +
 855                                  le32_to_cpu(hdr->header.ucode_array_offset_bytes));
 856                         table_offset = le32_to_cpu(hdr->jt_offset);
 857                         table_size = le32_to_cpu(hdr->jt_size);
 858                 } else  if (me == 4) {
 859                         const struct gfx_firmware_header_v1_0 *hdr =
 860                                 (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec2_fw->data;
 861                         fw_data = (const __le32 *)
 862                                 (adev->gfx.mec2_fw->data +
 863                                  le32_to_cpu(hdr->header.ucode_array_offset_bytes));
 864                         table_offset = le32_to_cpu(hdr->jt_offset);
 865                         table_size = le32_to_cpu(hdr->jt_size);
 866                 }
 867
 868                 for (i = 0; i < table_size; i ++) {
 869                         dst_ptr[bo_offset + i] =
 870                                 cpu_to_le32(le32_to_cpu(fw_data[table_offset + i]));
 871                 }
 872
 873                 bo_offset += table_size;
 874         }
 875 }
 876
 877 static void gfx_v9_0_rlc_fini(struct amdgpu_device *adev)
 878 {
 879         /* clear state block */
 880         amdgpu_bo_free_kernel(&adev->gfx.rlc.clear_state_obj,
 881                         &adev->gfx.rlc.clear_state_gpu_addr,
 882                         (void **)&adev->gfx.rlc.cs_ptr);
 883
 884         /* jump table block */
 885         amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj,
 886                         &adev->gfx.rlc.cp_table_gpu_addr,
 887                         (void **)&adev->gfx.rlc.cp_table_ptr);
 888 }
 889
 890 static int gfx_v9_0_rlc_init(struct amdgpu_device *adev)
 891 {
 892         volatile u32 *dst_ptr;
 893         u32 dws;
 894         const struct cs_section_def *cs_data;
 895         int r;
 896
 897         adev->gfx.rlc.cs_data = gfx9_cs_data;
 898
 899         cs_data = adev->gfx.rlc.cs_data;
 900
 901         if (cs_data) {
 902                 /* clear state block */
 903                 adev->gfx.rlc.clear_state_size = dws = gfx_v9_0_get_csb_size(adev);
 904                 r = amdgpu_bo_create_reserved(adev, dws * 4, PAGE_SIZE,
 905                                               AMDGPU_GEM_DOMAIN_VRAM,
 906                                               &adev->gfx.rlc.clear_state_obj,
 907                                               &adev->gfx.rlc.clear_state_gpu_addr,
 908                                               (void **)&adev->gfx.rlc.cs_ptr);
 909                 if (r) {
 910                         dev_err(adev->dev, "(%d) failed to create rlc csb bo\n",
 911                                 r);
 912                         gfx_v9_0_rlc_fini(adev);
 913                         return r;
 914                 }
 915                 /* set up the cs buffer */
 916                 dst_ptr = adev->gfx.rlc.cs_ptr;
 917                 gfx_v9_0_get_csb_buffer(adev, dst_ptr);
 918                 amdgpu_bo_kunmap(adev->gfx.rlc.clear_state_obj);
 919                 amdgpu_bo_unreserve(adev->gfx.rlc.clear_state_obj);
 920         }
 921
 922         if (adev->asic_type == CHIP_RAVEN) {
 923                 /* TODO: double check the cp_table_size for RV */
 924                 adev->gfx.rlc.cp_table_size = ALIGN(96 * 5 * 4, 2048) + (64 * 1024); /* JT + GDS */
 925                 r = amdgpu_bo_create_reserved(adev, adev->gfx.rlc.cp_table_size,
 926                                               PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
 927                                               &adev->gfx.rlc.cp_table_obj,
 928                                               &adev->gfx.rlc.cp_table_gpu_addr,
 929                                               (void **)&adev->gfx.rlc.cp_table_ptr);
 930                 if (r) {
 931                         dev_err(adev->dev,
 932                                 "(%d) failed to create cp table bo\n", r);
 933                         gfx_v9_0_rlc_fini(adev);
 934                         return r;
 935                 }
 936
 937                 rv_init_cp_jump_table(adev);
 938                 amdgpu_bo_kunmap(adev->gfx.rlc.cp_table_obj);
 939                 amdgpu_bo_unreserve(adev->gfx.rlc.cp_table_obj);
 940
 941                 gfx_v9_0_init_lbpw(adev);
 942         }
 943
 944         return 0;
 945 }
 946
 947 static void gfx_v9_0_mec_fini(struct amdgpu_device *adev)
 948 {
 949         amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL);
 950         amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_obj, NULL, NULL);
 951 }
 952
 953 static int gfx_v9_0_mec_init(struct amdgpu_device *adev)
 954 {
 955         int r;
 956         u32 *hpd;
 957         const __le32 *fw_data;
 958         unsigned fw_size;
 959         u32 *fw;
 960         size_t mec_hpd_size;
 961
 962         const struct gfx_firmware_header_v1_0 *mec_hdr;
 963
 964         bitmap_zero(adev->gfx.mec.queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
 965
 966         /* take ownership of the relevant compute queues */
 967         amdgpu_gfx_compute_queue_acquire(adev);
 968         mec_hpd_size = adev->gfx.num_compute_rings * GFX9_MEC_HPD_SIZE;
 969
 970         r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE,
 971                                       AMDGPU_GEM_DOMAIN_GTT,
 972                                       &adev->gfx.mec.hpd_eop_obj,
 973                                       &adev->gfx.mec.hpd_eop_gpu_addr,
 974                                       (void **)&hpd);
 975         if (r) {
 976                 dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r);
 977                 gfx_v9_0_mec_fini(adev);
 978                 return r;
 979         }
 980
 981         memset(hpd, 0, adev->gfx.mec.hpd_eop_obj->tbo.mem.size);
 982
 983         amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj);
 984         amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj);
 985
 986         mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
 987
 988         fw_data = (const __le32 *)
 989                 (adev->gfx.mec_fw->data +
 990                  le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
 991         fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes) / 4;
 992
 993         r = amdgpu_bo_create_reserved(adev, mec_hdr->header.ucode_size_bytes,
 994                                       PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
 995                                       &adev->gfx.mec.mec_fw_obj,
 996                                       &adev->gfx.mec.mec_fw_gpu_addr,
 997                                       (void **)&fw);
 998         if (r) {
 999                 dev_warn(adev->dev, "(%d) create mec firmware bo failed\n", r);
1000                 gfx_v9_0_mec_fini(adev);
1001                 return r;
1002         }
1003
1004         memcpy(fw, fw_data, fw_size);
1005
1006         amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj);
1007         amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj);
1008
1009         return 0;
1010 }
1011
1012 static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t address)
1013 {
1014         WREG32_SOC15(GC, 0, mmSQ_IND_INDEX,
1015                 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
1016                 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
1017                 (address << SQ_IND_INDEX__INDEX__SHIFT) |
1018                 (SQ_IND_INDEX__FORCE_READ_MASK));
1019         return RREG32_SOC15(GC, 0, mmSQ_IND_DATA);
1020 }
1021
1022 static void wave_read_regs(struct amdgpu_device *adev, uint32_t simd,
1023                            uint32_t wave, uint32_t thread,
1024                            uint32_t regno, uint32_t num, uint32_t *out)
1025 {
1026         WREG32_SOC15(GC, 0, mmSQ_IND_INDEX,
1027                 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
1028                 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
1029                 (regno << SQ_IND_INDEX__INDEX__SHIFT) |
1030                 (thread << SQ_IND_INDEX__THREAD_ID__SHIFT) |
1031                 (SQ_IND_INDEX__FORCE_READ_MASK) |
1032                 (SQ_IND_INDEX__AUTO_INCR_MASK));
1033         while (num--)
1034                 *(out++) = RREG32_SOC15(GC, 0, mmSQ_IND_DATA);
1035 }
1036
1037 static void gfx_v9_0_read_wave_data(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields)
1038 {
1039         /* type 1 wave data */
1040         dst[(*no_fields)++] = 1;
1041         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_STATUS);
1042         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_LO);
1043         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_HI);
1044         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_LO);
1045         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_HI);
1046         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_HW_ID);
1047         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW0);
1048         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW1);
1049         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_GPR_ALLOC);
1050         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_LDS_ALLOC);
1051         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TRAPSTS);
1052         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_STS);
1053         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_DBG0);
1054         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_M0);
1055 }
1056
1057 static void gfx_v9_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t simd,
1058                                      uint32_t wave, uint32_t start,
1059                                      uint32_t size, uint32_t *dst)
1060 {
1061         wave_read_regs(
1062                 adev, simd, wave, 0,
1063                 start + SQIND_WAVE_SGPRS_OFFSET, size, dst);
1064 }
1065
1066 static void gfx_v9_0_read_wave_vgprs(struct amdgpu_device *adev, uint32_t simd,
1067                                      uint32_t wave, uint32_t thread,
1068                                      uint32_t start, uint32_t size,
1069                                      uint32_t *dst)
1070 {
1071         wave_read_regs(
1072                 adev, simd, wave, thread,
1073                 start + SQIND_WAVE_VGPRS_OFFSET, size, dst);
1074 }
1075
1076 static void gfx_v9_0_select_me_pipe_q(struct amdgpu_device *adev,
1077                                   u32 me, u32 pipe, u32 q)
1078 {
1079         soc15_grbm_select(adev, me, pipe, q, 0);
1080 }
1081
1082 static const struct amdgpu_gfx_funcs gfx_v9_0_gfx_funcs = {
1083         .get_gpu_clock_counter = &gfx_v9_0_get_gpu_clock_counter,
1084         .select_se_sh = &gfx_v9_0_select_se_sh,
1085         .read_wave_data = &gfx_v9_0_read_wave_data,
1086         .read_wave_sgprs = &gfx_v9_0_read_wave_sgprs,
1087         .read_wave_vgprs = &gfx_v9_0_read_wave_vgprs,
1088         .select_me_pipe_q = &gfx_v9_0_select_me_pipe_q
1089 };
1090
1091 static void gfx_v9_0_gpu_early_init(struct amdgpu_device *adev)
1092 {
1093         u32 gb_addr_config;
1094
1095         adev->gfx.funcs = &gfx_v9_0_gfx_funcs;
1096
1097         switch (adev->asic_type) {
1098         case CHIP_VEGA10:
1099                 adev->gfx.config.max_hw_contexts = 8;
1100                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1101                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1102                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1103                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
1104                 gb_addr_config = VEGA10_GB_ADDR_CONFIG_GOLDEN;
1105                 break;
1106         case CHIP_VEGA12:
1107                 adev->gfx.config.max_hw_contexts = 8;
1108                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1109                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1110                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1111                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
1112                 gb_addr_config = VEGA12_GB_ADDR_CONFIG_GOLDEN;
1113                 DRM_INFO("fix gfx.config for vega12\n");
1114                 break;
1115         case CHIP_RAVEN:
1116                 adev->gfx.config.max_hw_contexts = 8;
1117                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1118                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1119                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1120                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
1121                 gb_addr_config = RAVEN_GB_ADDR_CONFIG_GOLDEN;
1122                 break;
1123         default:
1124                 BUG();
1125                 break;
1126         }
1127
1128         adev->gfx.config.gb_addr_config = gb_addr_config;
1129
1130         adev->gfx.config.gb_addr_config_fields.num_pipes = 1 <<
1131                         REG_GET_FIELD(
1132                                         adev->gfx.config.gb_addr_config,
1133                                         GB_ADDR_CONFIG,
1134                                         NUM_PIPES);
1135
1136         adev->gfx.config.max_tile_pipes =
1137                 adev->gfx.config.gb_addr_config_fields.num_pipes;
1138
1139         adev->gfx.config.gb_addr_config_fields.num_banks = 1 <<
1140                         REG_GET_FIELD(
1141                                         adev->gfx.config.gb_addr_config,
1142                                         GB_ADDR_CONFIG,
1143                                         NUM_BANKS);
1144         adev->gfx.config.gb_addr_config_fields.max_compress_frags = 1 <<
1145                         REG_GET_FIELD(
1146                                         adev->gfx.config.gb_addr_config,
1147                                         GB_ADDR_CONFIG,
1148                                         MAX_COMPRESSED_FRAGS);
1149         adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 1 <<
1150                         REG_GET_FIELD(
1151                                         adev->gfx.config.gb_addr_config,
1152                                         GB_ADDR_CONFIG,
1153                                         NUM_RB_PER_SE);
1154         adev->gfx.config.gb_addr_config_fields.num_se = 1 <<
1155                         REG_GET_FIELD(
1156                                         adev->gfx.config.gb_addr_config,
1157                                         GB_ADDR_CONFIG,
1158                                         NUM_SHADER_ENGINES);
1159         adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 1 << (8 +
1160                         REG_GET_FIELD(
1161                                         adev->gfx.config.gb_addr_config,
1162                                         GB_ADDR_CONFIG,
1163                                         PIPE_INTERLEAVE_SIZE));
1164 }
1165
1166 static int gfx_v9_0_ngg_create_buf(struct amdgpu_device *adev,
1167                                    struct amdgpu_ngg_buf *ngg_buf,
1168                                    int size_se,
1169                                    int default_size_se)
1170 {
1171         int r;
1172
1173         if (size_se < 0) {
1174                 dev_err(adev->dev, "Buffer size is invalid: %d\n", size_se);
1175                 return -EINVAL;
1176         }
1177         size_se = size_se ? size_se : default_size_se;
1178
1179         ngg_buf->size = size_se * adev->gfx.config.max_shader_engines;
1180         r = amdgpu_bo_create_kernel(adev, ngg_buf->size,
1181                                     PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
1182                                     &ngg_buf->bo,
1183                                     &ngg_buf->gpu_addr,
1184                                     NULL);
1185         if (r) {
1186                 dev_err(adev->dev, "(%d) failed to create NGG buffer\n", r);
1187                 return r;
1188         }
1189         ngg_buf->bo_size = amdgpu_bo_size(ngg_buf->bo);
1190
1191         return r;
1192 }
1193
1194 static int gfx_v9_0_ngg_fini(struct amdgpu_device *adev)
1195 {
1196         int i;
1197
1198         for (i = 0; i < NGG_BUF_MAX; i++)
1199                 amdgpu_bo_free_kernel(&adev->gfx.ngg.buf[i].bo,
1200                                       &adev->gfx.ngg.buf[i].gpu_addr,
1201                                       NULL);
1202
1203         memset(&adev->gfx.ngg.buf[0], 0,
1204                         sizeof(struct amdgpu_ngg_buf) * NGG_BUF_MAX);
1205
1206         adev->gfx.ngg.init = false;
1207
1208         return 0;
1209 }
1210
1211 static int gfx_v9_0_ngg_init(struct amdgpu_device *adev)
1212 {
1213         int r;
1214
1215         if (!amdgpu_ngg || adev->gfx.ngg.init == true)
1216                 return 0;
1217
1218         /* GDS reserve memory: 64 bytes alignment */
1219         adev->gfx.ngg.gds_reserve_size = ALIGN(5 * 4, 0x40);
1220         adev->gds.mem.total_size -= adev->gfx.ngg.gds_reserve_size;
1221         adev->gds.mem.gfx_partition_size -= adev->gfx.ngg.gds_reserve_size;
1222         adev->gfx.ngg.gds_reserve_addr = RREG32_SOC15(GC, 0, mmGDS_VMID0_BASE);
1223         adev->gfx.ngg.gds_reserve_addr += RREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE);
1224
1225         /* Primitive Buffer */
1226         r = gfx_v9_0_ngg_create_buf(adev, &adev->gfx.ngg.buf[NGG_PRIM],
1227                                     amdgpu_prim_buf_per_se,
1228                                     64 * 1024);
1229         if (r) {
1230                 dev_err(adev->dev, "Failed to create Primitive Buffer\n");
1231                 goto err;
1232         }
1233
1234         /* Position Buffer */
1235         r = gfx_v9_0_ngg_create_buf(adev, &adev->gfx.ngg.buf[NGG_POS],
1236                                     amdgpu_pos_buf_per_se,
1237                                     256 * 1024);
1238         if (r) {
1239                 dev_err(adev->dev, "Failed to create Position Buffer\n");
1240                 goto err;
1241         }
1242
1243         /* Control Sideband */
1244         r = gfx_v9_0_ngg_create_buf(adev, &adev->gfx.ngg.buf[NGG_CNTL],
1245                                     amdgpu_cntl_sb_buf_per_se,
1246                                     256);
1247         if (r) {
1248                 dev_err(adev->dev, "Failed to create Control Sideband Buffer\n");
1249                 goto err;
1250         }
1251
1252         /* Parameter Cache, not created by default */
1253         if (amdgpu_param_buf_per_se <= 0)
1254                 goto out;
1255
1256         r = gfx_v9_0_ngg_create_buf(adev, &adev->gfx.ngg.buf[NGG_PARAM],
1257                                     amdgpu_param_buf_per_se,
1258                                     512 * 1024);
1259         if (r) {
1260                 dev_err(adev->dev, "Failed to create Parameter Cache\n");
1261                 goto err;
1262         }
1263
1264 out:
1265         adev->gfx.ngg.init = true;
1266         return 0;
1267 err:
1268         gfx_v9_0_ngg_fini(adev);
1269         return r;
1270 }
1271
1272 static int gfx_v9_0_ngg_en(struct amdgpu_device *adev)
1273 {
1274         struct amdgpu_ring *ring = &adev->gfx.gfx_ring[0];
1275         int r;
1276         u32 data, base;
1277
1278         if (!amdgpu_ngg)
1279                 return 0;
1280
1281         /* Program buffer size */
1282         data = REG_SET_FIELD(0, WD_BUF_RESOURCE_1, INDEX_BUF_SIZE,
1283                              adev->gfx.ngg.buf[NGG_PRIM].size >> 8);
1284         data = REG_SET_FIELD(data, WD_BUF_RESOURCE_1, POS_BUF_SIZE,
1285                              adev->gfx.ngg.buf[NGG_POS].size >> 8);
1286         WREG32_SOC15(GC, 0, mmWD_BUF_RESOURCE_1, data);
1287
1288         data = REG_SET_FIELD(0, WD_BUF_RESOURCE_2, CNTL_SB_BUF_SIZE,
1289                              adev->gfx.ngg.buf[NGG_CNTL].size >> 8);
1290         data = REG_SET_FIELD(data, WD_BUF_RESOURCE_2, PARAM_BUF_SIZE,
1291                              adev->gfx.ngg.buf[NGG_PARAM].size >> 10);
1292         WREG32_SOC15(GC, 0, mmWD_BUF_RESOURCE_2, data);
1293
1294         /* Program buffer base address */
1295         base = lower_32_bits(adev->gfx.ngg.buf[NGG_PRIM].gpu_addr);
1296         data = REG_SET_FIELD(0, WD_INDEX_BUF_BASE, BASE, base);
1297         WREG32_SOC15(GC, 0, mmWD_INDEX_BUF_BASE, data);
1298
1299         base = upper_32_bits(adev->gfx.ngg.buf[NGG_PRIM].gpu_addr);
1300         data = REG_SET_FIELD(0, WD_INDEX_BUF_BASE_HI, BASE_HI, base);
1301         WREG32_SOC15(GC, 0, mmWD_INDEX_BUF_BASE_HI, data);
1302
1303         base = lower_32_bits(adev->gfx.ngg.buf[NGG_POS].gpu_addr);
1304         data = REG_SET_FIELD(0, WD_POS_BUF_BASE, BASE, base);
1305         WREG32_SOC15(GC, 0, mmWD_POS_BUF_BASE, data);
1306
1307         base = upper_32_bits(adev->gfx.ngg.buf[NGG_POS].gpu_addr);
1308         data = REG_SET_FIELD(0, WD_POS_BUF_BASE_HI, BASE_HI, base);
1309         WREG32_SOC15(GC, 0, mmWD_POS_BUF_BASE_HI, data);
1310
1311         base = lower_32_bits(adev->gfx.ngg.buf[NGG_CNTL].gpu_addr);
1312         data = REG_SET_FIELD(0, WD_CNTL_SB_BUF_BASE, BASE, base);
1313         WREG32_SOC15(GC, 0, mmWD_CNTL_SB_BUF_BASE, data);
1314
1315         base = upper_32_bits(adev->gfx.ngg.buf[NGG_CNTL].gpu_addr);
1316         data = REG_SET_FIELD(0, WD_CNTL_SB_BUF_BASE_HI, BASE_HI, base);
1317         WREG32_SOC15(GC, 0, mmWD_CNTL_SB_BUF_BASE_HI, data);
1318
1319         /* Clear GDS reserved memory */
1320         r = amdgpu_ring_alloc(ring, 17);
1321         if (r) {
1322                 DRM_ERROR("amdgpu: NGG failed to lock ring %d (%d).\n",
1323                           ring->idx, r);
1324                 return r;
1325         }
1326
1327         gfx_v9_0_write_data_to_reg(ring, 0, false,
1328                                    SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_SIZE),
1329                                    (adev->gds.mem.total_size +
1330                                     adev->gfx.ngg.gds_reserve_size) >>
1331                                    AMDGPU_GDS_SHIFT);
1332
1333         amdgpu_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
1334         amdgpu_ring_write(ring, (PACKET3_DMA_DATA_CP_SYNC |
1335                                 PACKET3_DMA_DATA_DST_SEL(1) |
1336                                 PACKET3_DMA_DATA_SRC_SEL(2)));
1337         amdgpu_ring_write(ring, 0);
1338         amdgpu_ring_write(ring, 0);
1339         amdgpu_ring_write(ring, adev->gfx.ngg.gds_reserve_addr);
1340         amdgpu_ring_write(ring, 0);
1341         amdgpu_ring_write(ring, PACKET3_DMA_DATA_CMD_RAW_WAIT |
1342                                 adev->gfx.ngg.gds_reserve_size);
1343
1344         gfx_v9_0_write_data_to_reg(ring, 0, false,
1345                                    SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_SIZE), 0);
1346
1347         amdgpu_ring_commit(ring);
1348
1349         return 0;
1350 }
1351
1352 static int gfx_v9_0_compute_ring_init(struct amdgpu_device *adev, int ring_id,
1353                                       int mec, int pipe, int queue)
1354 {
1355         int r;
1356         unsigned irq_type;
1357         struct amdgpu_ring *ring = &adev->gfx.compute_ring[ring_id];
1358
1359         ring = &adev->gfx.compute_ring[ring_id];
1360
1361         /* mec0 is me1 */
1362         ring->me = mec + 1;
1363         ring->pipe = pipe;
1364         ring->queue = queue;
1365
1366         ring->ring_obj = NULL;
1367         ring->use_doorbell = true;
1368         ring->doorbell_index = (AMDGPU_DOORBELL_MEC_RING0 + ring_id) << 1;
1369         ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr
1370                                 + (ring_id * GFX9_MEC_HPD_SIZE);
1371         sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue);
1372
1373         irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
1374                 + ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec)
1375                 + ring->pipe;
1376
1377         /* type-2 packets are deprecated on MEC, use type-3 instead */
1378         r = amdgpu_ring_init(adev, ring, 1024,
1379                              &adev->gfx.eop_irq, irq_type);
1380         if (r)
1381                 return r;
1382
1383
1384         return 0;
1385 }
1386
1387 static int gfx_v9_0_sw_init(void *handle)
1388 {
1389         int i, j, k, r, ring_id;
1390         struct amdgpu_ring *ring;
1391         struct amdgpu_kiq *kiq;
1392         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1393
1394         switch (adev->asic_type) {
1395         case CHIP_VEGA10:
1396         case CHIP_VEGA12:
1397         case CHIP_RAVEN:
1398                 adev->gfx.mec.num_mec = 2;
1399                 break;
1400         default:
1401                 adev->gfx.mec.num_mec = 1;
1402                 break;
1403         }
1404
1405         adev->gfx.mec.num_pipe_per_mec = 4;
1406         adev->gfx.mec.num_queue_per_pipe = 8;
1407
1408         /* KIQ event */
1409         r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, 178, &adev->gfx.kiq.irq);
1410         if (r)
1411                 return r;
1412
1413         /* EOP Event */
1414         r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, 181, &adev->gfx.eop_irq);
1415         if (r)
1416                 return r;
1417
1418         /* Privileged reg */
1419         r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, 184,
1420                               &adev->gfx.priv_reg_irq);
1421         if (r)
1422                 return r;
1423
1424         /* Privileged inst */
1425         r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, 185,
1426                               &adev->gfx.priv_inst_irq);
1427         if (r)
1428                 return r;
1429
1430         adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE;
1431
1432         gfx_v9_0_scratch_init(adev);
1433
1434         r = gfx_v9_0_init_microcode(adev);
1435         if (r) {
1436                 DRM_ERROR("Failed to load gfx firmware!\n");
1437                 return r;
1438         }
1439
1440         r = gfx_v9_0_rlc_init(adev);
1441         if (r) {
1442                 DRM_ERROR("Failed to init rlc BOs!\n");
1443                 return r;
1444         }
1445
1446         r = gfx_v9_0_mec_init(adev);
1447         if (r) {
1448                 DRM_ERROR("Failed to init MEC BOs!\n");
1449                 return r;
1450         }
1451
1452         /* set up the gfx ring */
1453         for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
1454                 ring = &adev->gfx.gfx_ring[i];
1455                 ring->ring_obj = NULL;
1456                 if (!i)
1457                         sprintf(ring->name, "gfx");
1458                 else
1459                         sprintf(ring->name, "gfx_%d", i);
1460                 ring->use_doorbell = true;
1461                 ring->doorbell_index = AMDGPU_DOORBELL64_GFX_RING0 << 1;
1462                 r = amdgpu_ring_init(adev, ring, 1024,
1463                                      &adev->gfx.eop_irq, AMDGPU_CP_IRQ_GFX_EOP);
1464                 if (r)
1465                         return r;
1466         }
1467
1468         /* set up the compute queues - allocate horizontally across pipes */
1469         ring_id = 0;
1470         for (i = 0; i < adev->gfx.mec.num_mec; ++i) {
1471                 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {
1472                         for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {
1473                                 if (!amdgpu_gfx_is_mec_queue_enabled(adev, i, k, j))
1474                                         continue;
1475
1476                                 r = gfx_v9_0_compute_ring_init(adev,
1477                                                                ring_id,
1478                                                                i, k, j);
1479                                 if (r)
1480                                         return r;
1481
1482                                 ring_id++;
1483                         }
1484                 }
1485         }
1486
1487         r = amdgpu_gfx_kiq_init(adev, GFX9_MEC_HPD_SIZE);
1488         if (r) {
1489                 DRM_ERROR("Failed to init KIQ BOs!\n");
1490                 return r;
1491         }
1492
1493         kiq = &adev->gfx.kiq;
1494         r = amdgpu_gfx_kiq_init_ring(adev, &kiq->ring, &kiq->irq);
1495         if (r)
1496                 return r;
1497
1498         /* create MQD for all compute queues as wel as KIQ for SRIOV case */
1499         r = amdgpu_gfx_compute_mqd_sw_init(adev, sizeof(struct v9_mqd_allocation));
1500         if (r)
1501                 return r;
1502
1503         /* reserve GDS, GWS and OA resource for gfx */
1504         r = amdgpu_bo_create_kernel(adev, adev->gds.mem.gfx_partition_size,
1505                                     PAGE_SIZE, AMDGPU_GEM_DOMAIN_GDS,
1506                                     &adev->gds.gds_gfx_bo, NULL, NULL);
1507         if (r)
1508                 return r;
1509
1510         r = amdgpu_bo_create_kernel(adev, adev->gds.gws.gfx_partition_size,
1511                                     PAGE_SIZE, AMDGPU_GEM_DOMAIN_GWS,
1512                                     &adev->gds.gws_gfx_bo, NULL, NULL);
1513         if (r)
1514                 return r;
1515
1516         r = amdgpu_bo_create_kernel(adev, adev->gds.oa.gfx_partition_size,
1517                                     PAGE_SIZE, AMDGPU_GEM_DOMAIN_OA,
1518                                     &adev->gds.oa_gfx_bo, NULL, NULL);
1519         if (r)
1520                 return r;
1521
1522         adev->gfx.ce_ram_size = 0x8000;
1523
1524         gfx_v9_0_gpu_early_init(adev);
1525
1526         r = gfx_v9_0_ngg_init(adev);
1527         if (r)
1528                 return r;
1529
1530         return 0;
1531 }
1532
1533
1534 static int gfx_v9_0_sw_fini(void *handle)
1535 {
1536         int i;
1537         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1538
1539         amdgpu_bo_free_kernel(&adev->gds.oa_gfx_bo, NULL, NULL);
1540         amdgpu_bo_free_kernel(&adev->gds.gws_gfx_bo, NULL, NULL);
1541         amdgpu_bo_free_kernel(&adev->gds.gds_gfx_bo, NULL, NULL);
1542
1543         for (i = 0; i < adev->gfx.num_gfx_rings; i++)
1544                 amdgpu_ring_fini(&adev->gfx.gfx_ring[i]);
1545         for (i = 0; i < adev->gfx.num_compute_rings; i++)
1546                 amdgpu_ring_fini(&adev->gfx.compute_ring[i]);
1547
1548         amdgpu_gfx_compute_mqd_sw_fini(adev);
1549         amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq.ring, &adev->gfx.kiq.irq);
1550         amdgpu_gfx_kiq_fini(adev);
1551
1552         gfx_v9_0_mec_fini(adev);
1553         gfx_v9_0_ngg_fini(adev);
1554         amdgpu_bo_free_kernel(&adev->gfx.rlc.clear_state_obj,
1555                                 &adev->gfx.rlc.clear_state_gpu_addr,
1556                                 (void **)&adev->gfx.rlc.cs_ptr);
1557         if (adev->asic_type == CHIP_RAVEN) {
1558                 amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj,
1559                                 &adev->gfx.rlc.cp_table_gpu_addr,
1560                                 (void **)&adev->gfx.rlc.cp_table_ptr);
1561         }
1562         gfx_v9_0_free_microcode(adev);
1563
1564         return 0;
1565 }
1566
1567
1568 static void gfx_v9_0_tiling_mode_table_init(struct amdgpu_device *adev)
1569 {
1570         /* TODO */
1571 }
1572
1573 static void gfx_v9_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num, u32 instance)
1574 {
1575         u32 data;
1576
1577         if (instance == 0xffffffff)
1578                 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES, 1);
1579         else
1580                 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX, instance);
1581
1582         if (se_num == 0xffffffff)
1583                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES, 1);
1584         else
1585                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num);
1586
1587         if (sh_num == 0xffffffff)
1588                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_BROADCAST_WRITES, 1);
1589         else
1590                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_INDEX, sh_num);
1591
1592         WREG32_SOC15(GC, 0, mmGRBM_GFX_INDEX, data);
1593 }
1594
1595 static u32 gfx_v9_0_get_rb_active_bitmap(struct amdgpu_device *adev)
1596 {
1597         u32 data, mask;
1598
1599         data = RREG32_SOC15(GC, 0, mmCC_RB_BACKEND_DISABLE);
1600         data |= RREG32_SOC15(GC, 0, mmGC_USER_RB_BACKEND_DISABLE);
1601
1602         data &= CC_RB_BACKEND_DISABLE__BACKEND_DISABLE_MASK;
1603         data >>= GC_USER_RB_BACKEND_DISABLE__BACKEND_DISABLE__SHIFT;
1604
1605         mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se /
1606                                          adev->gfx.config.max_sh_per_se);
1607
1608         return (~data) & mask;
1609 }
1610
1611 static void gfx_v9_0_setup_rb(struct amdgpu_device *adev)
1612 {
1613         int i, j;
1614         u32 data;
1615         u32 active_rbs = 0;
1616         u32 rb_bitmap_width_per_sh = adev->gfx.config.max_backends_per_se /
1617                                         adev->gfx.config.max_sh_per_se;
1618
1619         mutex_lock(&adev->grbm_idx_mutex);
1620         for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
1621                 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
1622                         gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff);
1623                         data = gfx_v9_0_get_rb_active_bitmap(adev);
1624                         active_rbs |= data << ((i * adev->gfx.config.max_sh_per_se + j) *
1625                                                rb_bitmap_width_per_sh);
1626                 }
1627         }
1628         gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
1629         mutex_unlock(&adev->grbm_idx_mutex);
1630
1631         adev->gfx.config.backend_enable_mask = active_rbs;
1632         adev->gfx.config.num_rbs = hweight32(active_rbs);
1633 }
1634
1635 #define DEFAULT_SH_MEM_BASES    (0x6000)
1636 #define FIRST_COMPUTE_VMID      (8)
1637 #define LAST_COMPUTE_VMID       (16)
1638 static void gfx_v9_0_init_compute_vmid(struct amdgpu_device *adev)
1639 {
1640         int i;
1641         uint32_t sh_mem_config;
1642         uint32_t sh_mem_bases;
1643
1644         /*
1645          * Configure apertures:
1646          * LDS:         0x60000000'00000000 - 0x60000001'00000000 (4GB)
1647          * Scratch:     0x60000001'00000000 - 0x60000002'00000000 (4GB)
1648          * GPUVM:       0x60010000'00000000 - 0x60020000'00000000 (1TB)
1649          */
1650         sh_mem_bases = DEFAULT_SH_MEM_BASES | (DEFAULT_SH_MEM_BASES << 16);
1651
1652         sh_mem_config = SH_MEM_ADDRESS_MODE_64 |
1653                         SH_MEM_ALIGNMENT_MODE_UNALIGNED <<
1654                         SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT;
1655
1656         mutex_lock(&adev->srbm_mutex);
1657         for (i = FIRST_COMPUTE_VMID; i < LAST_COMPUTE_VMID; i++) {
1658                 soc15_grbm_select(adev, 0, 0, 0, i);
1659                 /* CP and shaders */
1660                 WREG32_SOC15(GC, 0, mmSH_MEM_CONFIG, sh_mem_config);
1661                 WREG32_SOC15(GC, 0, mmSH_MEM_BASES, sh_mem_bases);
1662         }
1663         soc15_grbm_select(adev, 0, 0, 0, 0);
1664         mutex_unlock(&adev->srbm_mutex);
1665 }
1666
1667 static void gfx_v9_0_gpu_init(struct amdgpu_device *adev)
1668 {
1669         u32 tmp;
1670         int i;
1671
1672         WREG32_FIELD15(GC, 0, GRBM_CNTL, READ_TIMEOUT, 0xff);
1673
1674         gfx_v9_0_tiling_mode_table_init(adev);
1675
1676         gfx_v9_0_setup_rb(adev);
1677         gfx_v9_0_get_cu_info(adev, &adev->gfx.cu_info);
1678         adev->gfx.config.db_debug2 = RREG32_SOC15(GC, 0, mmDB_DEBUG2);
1679
1680         /* XXX SH_MEM regs */
1681         /* where to put LDS, scratch, GPUVM in FSA64 space */
1682         mutex_lock(&adev->srbm_mutex);
1683         for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB].num_ids; i++) {
1684                 soc15_grbm_select(adev, 0, 0, 0, i);
1685                 /* CP and shaders */
1686                 if (i == 0) {
1687                         tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE,
1688                                             SH_MEM_ALIGNMENT_MODE_UNALIGNED);
1689                         WREG32_SOC15(GC, 0, mmSH_MEM_CONFIG, tmp);
1690                         WREG32_SOC15(GC, 0, mmSH_MEM_BASES, 0);
1691                 } else {
1692                         tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE,
1693                                             SH_MEM_ALIGNMENT_MODE_UNALIGNED);
1694                         WREG32_SOC15(GC, 0, mmSH_MEM_CONFIG, tmp);
1695                         tmp = REG_SET_FIELD(0, SH_MEM_BASES, PRIVATE_BASE,
1696                                 (adev->gmc.private_aperture_start >> 48));
1697                         tmp = REG_SET_FIELD(tmp, SH_MEM_BASES, SHARED_BASE,
1698                                 (adev->gmc.shared_aperture_start >> 48));
1699                         WREG32_SOC15(GC, 0, mmSH_MEM_BASES, tmp);
1700                 }
1701         }
1702         soc15_grbm_select(adev, 0, 0, 0, 0);
1703
1704         mutex_unlock(&adev->srbm_mutex);
1705
1706         gfx_v9_0_init_compute_vmid(adev);
1707
1708         mutex_lock(&adev->grbm_idx_mutex);
1709         /*
1710          * making sure that the following register writes will be broadcasted
1711          * to all the shaders
1712          */
1713         gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
1714
1715         WREG32_SOC15(GC, 0, mmPA_SC_FIFO_SIZE,
1716                    (adev->gfx.config.sc_prim_fifo_size_frontend <<
1717                         PA_SC_FIFO_SIZE__SC_FRONTEND_PRIM_FIFO_SIZE__SHIFT) |
1718                    (adev->gfx.config.sc_prim_fifo_size_backend <<
1719                         PA_SC_FIFO_SIZE__SC_BACKEND_PRIM_FIFO_SIZE__SHIFT) |
1720                    (adev->gfx.config.sc_hiz_tile_fifo_size <<
1721                         PA_SC_FIFO_SIZE__SC_HIZ_TILE_FIFO_SIZE__SHIFT) |
1722                    (adev->gfx.config.sc_earlyz_tile_fifo_size <<
1723                         PA_SC_FIFO_SIZE__SC_EARLYZ_TILE_FIFO_SIZE__SHIFT));
1724         mutex_unlock(&adev->grbm_idx_mutex);
1725
1726 }
1727
1728 static void gfx_v9_0_wait_for_rlc_serdes(struct amdgpu_device *adev)
1729 {
1730         u32 i, j, k;
1731         u32 mask;
1732
1733         mutex_lock(&adev->grbm_idx_mutex);
1734         for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
1735                 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
1736                         gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff);
1737                         for (k = 0; k < adev->usec_timeout; k++) {
1738                                 if (RREG32_SOC15(GC, 0, mmRLC_SERDES_CU_MASTER_BUSY) == 0)
1739                                         break;
1740                                 udelay(1);
1741                         }
1742                         if (k == adev->usec_timeout) {
1743                                 gfx_v9_0_select_se_sh(adev, 0xffffffff,
1744                                                       0xffffffff, 0xffffffff);
1745                                 mutex_unlock(&adev->grbm_idx_mutex);
1746                                 DRM_INFO("Timeout wait for RLC serdes %u,%u\n",
1747                                          i, j);
1748                                 return;
1749                         }
1750                 }
1751         }
1752         gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
1753         mutex_unlock(&adev->grbm_idx_mutex);
1754
1755         mask = RLC_SERDES_NONCU_MASTER_BUSY__SE_MASTER_BUSY_MASK |
1756                 RLC_SERDES_NONCU_MASTER_BUSY__GC_MASTER_BUSY_MASK |
1757                 RLC_SERDES_NONCU_MASTER_BUSY__TC0_MASTER_BUSY_MASK |
1758                 RLC_SERDES_NONCU_MASTER_BUSY__TC1_MASTER_BUSY_MASK;
1759         for (k = 0; k < adev->usec_timeout; k++) {
1760                 if ((RREG32_SOC15(GC, 0, mmRLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
1761                         break;
1762                 udelay(1);
1763         }
1764 }
1765
1766 static void gfx_v9_0_enable_gui_idle_interrupt(struct amdgpu_device *adev,
1767                                                bool enable)
1768 {
1769         u32 tmp = RREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0);
1770
1771         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE, enable ? 1 : 0);
1772         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE, enable ? 1 : 0);
1773         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE, enable ? 1 : 0);
1774         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE, enable ? 1 : 0);
1775
1776         WREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0, tmp);
1777 }
1778
1779 static void gfx_v9_0_init_csb(struct amdgpu_device *adev)
1780 {
1781         /* csib */
1782         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_HI),
1783                         adev->gfx.rlc.clear_state_gpu_addr >> 32);
1784         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_LO),
1785                         adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc);
1786         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_LENGTH),
1787                         adev->gfx.rlc.clear_state_size);
1788 }
1789
1790 static void gfx_v9_1_parse_ind_reg_list(int *register_list_format,
1791                                 int indirect_offset,
1792                                 int list_size,
1793                                 int *unique_indirect_regs,
1794                                 int *unique_indirect_reg_count,
1795                                 int *indirect_start_offsets,
1796                                 int *indirect_start_offsets_count)
1797 {
1798         int idx;
1799
1800         for (; indirect_offset < list_size; indirect_offset++) {
1801                 indirect_start_offsets[*indirect_start_offsets_count] = indirect_offset;
1802                 *indirect_start_offsets_count = *indirect_start_offsets_count + 1;
1803
1804                 while (register_list_format[indirect_offset] != 0xFFFFFFFF) {
1805                         indirect_offset += 2;
1806
1807                         /* look for the matching indice */
1808                         for (idx = 0; idx < *unique_indirect_reg_count; idx++) {
1809                                 if (unique_indirect_regs[idx] ==
1810                                         register_list_format[indirect_offset] ||
1811                                         !unique_indirect_regs[idx])
1812                                         break;
1813                         }
1814
1815                         BUG_ON(idx >= *unique_indirect_reg_count);
1816
1817                         if (!unique_indirect_regs[idx])
1818                                 unique_indirect_regs[idx] = register_list_format[indirect_offset];
1819
1820                         indirect_offset++;
1821                 }
1822         }
1823 }
1824
1825 static int gfx_v9_1_init_rlc_save_restore_list(struct amdgpu_device *adev)
1826 {
1827         int unique_indirect_regs[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0};
1828         int unique_indirect_reg_count = 0;
1829
1830         int indirect_start_offsets[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0};
1831         int indirect_start_offsets_count = 0;
1832
1833         int list_size = 0;
1834         int i = 0, j = 0;
1835         u32 tmp = 0;
1836
1837         u32 *register_list_format =
1838                 kmalloc(adev->gfx.rlc.reg_list_format_size_bytes, GFP_KERNEL);
1839         if (!register_list_format)
1840                 return -ENOMEM;
1841         memcpy(register_list_format, adev->gfx.rlc.register_list_format,
1842                 adev->gfx.rlc.reg_list_format_size_bytes);
1843
1844         /* setup unique_indirect_regs array and indirect_start_offsets array */
1845         unique_indirect_reg_count = ARRAY_SIZE(unique_indirect_regs);
1846         gfx_v9_1_parse_ind_reg_list(register_list_format,
1847                                     adev->gfx.rlc.reg_list_format_direct_reg_list_length,
1848                                     adev->gfx.rlc.reg_list_format_size_bytes >> 2,
1849                                     unique_indirect_regs,
1850                                     &unique_indirect_reg_count,
1851                                     indirect_start_offsets,
1852                                     &indirect_start_offsets_count);
1853
1854         /* enable auto inc in case it is disabled */
1855         tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL));
1856         tmp |= RLC_SRM_CNTL__AUTO_INCR_ADDR_MASK;
1857         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL), tmp);
1858
1859         /* write register_restore table to offset 0x0 using RLC_SRM_ARAM_ADDR/DATA */
1860         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_ADDR),
1861                 RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET);
1862         for (i = 0; i < adev->gfx.rlc.reg_list_size_bytes >> 2; i++)
1863                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_DATA),
1864                         adev->gfx.rlc.register_restore[i]);
1865
1866         /* load indirect register */
1867         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR),
1868                 adev->gfx.rlc.reg_list_format_start);
1869
1870         /* direct register portion */
1871         for (i = 0; i < adev->gfx.rlc.reg_list_format_direct_reg_list_length; i++)
1872                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA),
1873                         register_list_format[i]);
1874
1875         /* indirect register portion */
1876         while (i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2)) {
1877                 if (register_list_format[i] == 0xFFFFFFFF) {
1878                         WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]);
1879                         continue;
1880                 }
1881
1882                 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]);
1883                 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]);
1884
1885                 for (j = 0; j < unique_indirect_reg_count; j++) {
1886                         if (register_list_format[i] == unique_indirect_regs[j]) {
1887                                 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, j);
1888                                 break;
1889                         }
1890                 }
1891
1892                 BUG_ON(j >= unique_indirect_reg_count);
1893
1894                 i++;
1895         }
1896
1897         /* set save/restore list size */
1898         list_size = adev->gfx.rlc.reg_list_size_bytes >> 2;
1899         list_size = list_size >> 1;
1900         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR),
1901                 adev->gfx.rlc.reg_restore_list_size);
1902         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA), list_size);
1903
1904         /* write the starting offsets to RLC scratch ram */
1905         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR),
1906                 adev->gfx.rlc.starting_offsets_start);
1907         for (i = 0; i < ARRAY_SIZE(indirect_start_offsets); i++)
1908                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA),
1909                        indirect_start_offsets[i]);
1910
1911         /* load unique indirect regs*/
1912         for (i = 0; i < ARRAY_SIZE(unique_indirect_regs); i++) {
1913                 if (unique_indirect_regs[i] != 0) {
1914                         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_ADDR_0)
1915                                + GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[i],
1916                                unique_indirect_regs[i] & 0x3FFFF);
1917
1918                         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_DATA_0)
1919                                + GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[i],
1920                                unique_indirect_regs[i] >> 20);
1921                 }
1922         }
1923
1924         kfree(register_list_format);
1925         return 0;
1926 }
1927
1928 static void gfx_v9_0_enable_save_restore_machine(struct amdgpu_device *adev)
1929 {
1930         WREG32_FIELD15(GC, 0, RLC_SRM_CNTL, SRM_ENABLE, 1);
1931 }
1932
1933 static void pwr_10_0_gfxip_control_over_cgpg(struct amdgpu_device *adev,
1934                                              bool enable)
1935 {
1936         uint32_t data = 0;
1937         uint32_t default_data = 0;
1938
1939         default_data = data = RREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS));
1940         if (enable == true) {
1941                 /* enable GFXIP control over CGPG */
1942                 data |= PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK;
1943                 if(default_data != data)
1944                         WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data);
1945
1946                 /* update status */
1947                 data &= ~PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS_MASK;
1948                 data |= (2 << PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS__SHIFT);
1949                 if(default_data != data)
1950                         WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data);
1951         } else {
1952                 /* restore GFXIP control over GCPG */
1953                 data &= ~PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK;
1954                 if(default_data != data)
1955                         WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data);
1956         }
1957 }
1958
1959 static void gfx_v9_0_init_gfx_power_gating(struct amdgpu_device *adev)
1960 {
1961         uint32_t data = 0;
1962
1963         if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG |
1964                               AMD_PG_SUPPORT_GFX_SMG |
1965                               AMD_PG_SUPPORT_GFX_DMG)) {
1966                 /* init IDLE_POLL_COUNT = 60 */
1967                 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL));
1968                 data &= ~CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK;
1969                 data |= (0x60 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
1970                 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL), data);
1971
1972                 /* init RLC PG Delay */
1973                 data = 0;
1974                 data |= (0x10 << RLC_PG_DELAY__POWER_UP_DELAY__SHIFT);
1975                 data |= (0x10 << RLC_PG_DELAY__POWER_DOWN_DELAY__SHIFT);
1976                 data |= (0x10 << RLC_PG_DELAY__CMD_PROPAGATE_DELAY__SHIFT);
1977                 data |= (0x40 << RLC_PG_DELAY__MEM_SLEEP_DELAY__SHIFT);
1978                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY), data);
1979
1980                 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2));
1981                 data &= ~RLC_PG_DELAY_2__SERDES_CMD_DELAY_MASK;
1982                 data |= (0x4 << RLC_PG_DELAY_2__SERDES_CMD_DELAY__SHIFT);
1983                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2), data);
1984
1985                 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3));
1986                 data &= ~RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG_MASK;
1987                 data |= (0xff << RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG__SHIFT);
1988                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3), data);
1989
1990                 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL));
1991                 data &= ~RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD_MASK;
1992
1993                 /* program GRBM_REG_SAVE_GFX_IDLE_THRESHOLD to 0x55f0 */
1994                 data |= (0x55f0 << RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD__SHIFT);
1995                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL), data);
1996
1997                 pwr_10_0_gfxip_control_over_cgpg(adev, true);
1998         }
1999 }
2000
2001 static void gfx_v9_0_enable_sck_slow_down_on_power_up(struct amdgpu_device *adev,
2002                                                 bool enable)
2003 {
2004         uint32_t data = 0;
2005         uint32_t default_data = 0;
2006
2007         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2008         data = REG_SET_FIELD(data, RLC_PG_CNTL,
2009                              SMU_CLK_SLOWDOWN_ON_PU_ENABLE,
2010                              enable ? 1 : 0);
2011         if (default_data != data)
2012                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2013 }
2014
2015 static void gfx_v9_0_enable_sck_slow_down_on_power_down(struct amdgpu_device *adev,
2016                                                 bool enable)
2017 {
2018         uint32_t data = 0;
2019         uint32_t default_data = 0;
2020
2021         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2022         data = REG_SET_FIELD(data, RLC_PG_CNTL,
2023                              SMU_CLK_SLOWDOWN_ON_PD_ENABLE,
2024                              enable ? 1 : 0);
2025         if(default_data != data)
2026                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2027 }
2028
2029 static void gfx_v9_0_enable_cp_power_gating(struct amdgpu_device *adev,
2030                                         bool enable)
2031 {
2032         uint32_t data = 0;
2033         uint32_t default_data = 0;
2034
2035         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2036         data = REG_SET_FIELD(data, RLC_PG_CNTL,
2037                              CP_PG_DISABLE,
2038                              enable ? 0 : 1);
2039         if(default_data != data)
2040                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2041 }
2042
2043 static void gfx_v9_0_enable_gfx_cg_power_gating(struct amdgpu_device *adev,
2044                                                 bool enable)
2045 {
2046         uint32_t data, default_data;
2047
2048         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2049         data = REG_SET_FIELD(data, RLC_PG_CNTL,
2050                              GFX_POWER_GATING_ENABLE,
2051                              enable ? 1 : 0);
2052         if(default_data != data)
2053                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2054 }
2055
2056 static void gfx_v9_0_enable_gfx_pipeline_powergating(struct amdgpu_device *adev,
2057                                                 bool enable)
2058 {
2059         uint32_t data, default_data;
2060
2061         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2062         data = REG_SET_FIELD(data, RLC_PG_CNTL,
2063                              GFX_PIPELINE_PG_ENABLE,
2064                              enable ? 1 : 0);
2065         if(default_data != data)
2066                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2067
2068         if (!enable)
2069                 /* read any GFX register to wake up GFX */
2070                 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmDB_RENDER_CONTROL));
2071 }
2072
2073 static void gfx_v9_0_enable_gfx_static_mg_power_gating(struct amdgpu_device *adev,
2074                                                        bool enable)
2075 {
2076         uint32_t data, default_data;
2077
2078         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2079         data = REG_SET_FIELD(data, RLC_PG_CNTL,
2080                              STATIC_PER_CU_PG_ENABLE,
2081                              enable ? 1 : 0);
2082         if(default_data != data)
2083                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2084 }
2085
2086 static void gfx_v9_0_enable_gfx_dynamic_mg_power_gating(struct amdgpu_device *adev,
2087                                                 bool enable)
2088 {
2089         uint32_t data, default_data;
2090
2091         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2092         data = REG_SET_FIELD(data, RLC_PG_CNTL,
2093                              DYN_PER_CU_PG_ENABLE,
2094                              enable ? 1 : 0);
2095         if(default_data != data)
2096                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2097 }
2098
2099 static void gfx_v9_0_init_pg(struct amdgpu_device *adev)
2100 {
2101         if (!adev->gfx.rlc.is_rlc_v2_1)
2102                 return;
2103
2104         if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG |
2105                               AMD_PG_SUPPORT_GFX_SMG |
2106                               AMD_PG_SUPPORT_GFX_DMG |
2107                               AMD_PG_SUPPORT_CP |
2108                               AMD_PG_SUPPORT_GDS |
2109                               AMD_PG_SUPPORT_RLC_SMU_HS)) {
2110                 gfx_v9_0_init_csb(adev);
2111                 gfx_v9_1_init_rlc_save_restore_list(adev);
2112                 gfx_v9_0_enable_save_restore_machine(adev);
2113
2114                 WREG32(mmRLC_JUMP_TABLE_RESTORE,
2115                        adev->gfx.rlc.cp_table_gpu_addr >> 8);
2116                 gfx_v9_0_init_gfx_power_gating(adev);
2117         }
2118 }
2119
2120 void gfx_v9_0_rlc_stop(struct amdgpu_device *adev)
2121 {
2122         WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 0);
2123         gfx_v9_0_enable_gui_idle_interrupt(adev, false);
2124         gfx_v9_0_wait_for_rlc_serdes(adev);
2125 }
2126
2127 static void gfx_v9_0_rlc_reset(struct amdgpu_device *adev)
2128 {
2129         WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
2130         udelay(50);
2131         WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 0);
2132         udelay(50);
2133 }
2134
2135 static void gfx_v9_0_rlc_start(struct amdgpu_device *adev)
2136 {
2137 #ifdef AMDGPU_RLC_DEBUG_RETRY
2138         u32 rlc_ucode_ver;
2139 #endif
2140
2141         WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 1);
2142
2143         /* carrizo do enable cp interrupt after cp inited */
2144         if (!(adev->flags & AMD_IS_APU))
2145                 gfx_v9_0_enable_gui_idle_interrupt(adev, true);
2146
2147         udelay(50);
2148
2149 #ifdef AMDGPU_RLC_DEBUG_RETRY
2150         /* RLC_GPM_GENERAL_6 : RLC Ucode version */
2151         rlc_ucode_ver = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_6);
2152         if(rlc_ucode_ver == 0x108) {
2153                 DRM_INFO("Using rlc debug ucode. mmRLC_GPM_GENERAL_6 ==0x08%x / fw_ver == %i \n",
2154                                 rlc_ucode_ver, adev->gfx.rlc_fw_version);
2155                 /* RLC_GPM_TIMER_INT_3 : Timer interval in RefCLK cycles,
2156                  * default is 0x9C4 to create a 100us interval */
2157                 WREG32_SOC15(GC, 0, mmRLC_GPM_TIMER_INT_3, 0x9C4);
2158                 /* RLC_GPM_GENERAL_12 : Minimum gap between wptr and rptr
2159                  * to disable the page fault retry interrupts, default is
2160                  * 0x100 (256) */
2161                 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_12, 0x100);
2162         }
2163 #endif
2164 }
2165
2166 static int gfx_v9_0_rlc_load_microcode(struct amdgpu_device *adev)
2167 {
2168         const struct rlc_firmware_header_v2_0 *hdr;
2169         const __le32 *fw_data;
2170         unsigned i, fw_size;
2171
2172         if (!adev->gfx.rlc_fw)
2173                 return -EINVAL;
2174
2175         hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
2176         amdgpu_ucode_print_rlc_hdr(&hdr->header);
2177
2178         fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
2179                            le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2180         fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
2181
2182         WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR,
2183                         RLCG_UCODE_LOADING_START_ADDRESS);
2184         for (i = 0; i < fw_size; i++)
2185                 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++));
2186         WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR, adev->gfx.rlc_fw_version);
2187
2188         return 0;
2189 }
2190
2191 static int gfx_v9_0_rlc_resume(struct amdgpu_device *adev)
2192 {
2193         int r;
2194
2195         if (amdgpu_sriov_vf(adev)) {
2196                 gfx_v9_0_init_csb(adev);
2197                 return 0;
2198         }
2199
2200         gfx_v9_0_rlc_stop(adev);
2201
2202         /* disable CG */
2203         WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, 0);
2204
2205         /* disable PG */
2206         WREG32_SOC15(GC, 0, mmRLC_PG_CNTL, 0);
2207
2208         gfx_v9_0_rlc_reset(adev);
2209
2210         gfx_v9_0_init_pg(adev);
2211
2212         if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {
2213                 /* legacy rlc firmware loading */
2214                 r = gfx_v9_0_rlc_load_microcode(adev);
2215                 if (r)
2216                         return r;
2217         }
2218
2219         if (adev->asic_type == CHIP_RAVEN) {
2220                 if (amdgpu_lbpw != 0)
2221                         gfx_v9_0_enable_lbpw(adev, true);
2222                 else
2223                         gfx_v9_0_enable_lbpw(adev, false);
2224         }
2225
2226         gfx_v9_0_rlc_start(adev);
2227
2228         return 0;
2229 }
2230
2231 static void gfx_v9_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable)
2232 {
2233         int i;
2234         u32 tmp = RREG32_SOC15(GC, 0, mmCP_ME_CNTL);
2235
2236         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, enable ? 0 : 1);
2237         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, enable ? 0 : 1);
2238         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_HALT, enable ? 0 : 1);
2239         if (!enable) {
2240                 for (i = 0; i < adev->gfx.num_gfx_rings; i++)
2241                         adev->gfx.gfx_ring[i].ready = false;
2242         }
2243         WREG32_SOC15(GC, 0, mmCP_ME_CNTL, tmp);
2244         udelay(50);
2245 }
2246
2247 static int gfx_v9_0_cp_gfx_load_microcode(struct amdgpu_device *adev)
2248 {
2249         const struct gfx_firmware_header_v1_0 *pfp_hdr;
2250         const struct gfx_firmware_header_v1_0 *ce_hdr;
2251         const struct gfx_firmware_header_v1_0 *me_hdr;
2252         const __le32 *fw_data;
2253         unsigned i, fw_size;
2254
2255         if (!adev->gfx.me_fw || !adev->gfx.pfp_fw || !adev->gfx.ce_fw)
2256                 return -EINVAL;
2257
2258         pfp_hdr = (const struct gfx_firmware_header_v1_0 *)
2259                 adev->gfx.pfp_fw->data;
2260         ce_hdr = (const struct gfx_firmware_header_v1_0 *)
2261                 adev->gfx.ce_fw->data;
2262         me_hdr = (const struct gfx_firmware_header_v1_0 *)
2263                 adev->gfx.me_fw->data;
2264
2265         amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header);
2266         amdgpu_ucode_print_gfx_hdr(&ce_hdr->header);
2267         amdgpu_ucode_print_gfx_hdr(&me_hdr->header);
2268
2269         gfx_v9_0_cp_gfx_enable(adev, false);
2270
2271         /* PFP */
2272         fw_data = (const __le32 *)
2273                 (adev->gfx.pfp_fw->data +
2274                  le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
2275         fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
2276         WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, 0);
2277         for (i = 0; i < fw_size; i++)
2278                 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
2279         WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, adev->gfx.pfp_fw_version);
2280
2281         /* CE */
2282         fw_data = (const __le32 *)
2283                 (adev->gfx.ce_fw->data +
2284                  le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
2285         fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
2286         WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, 0);
2287         for (i = 0; i < fw_size; i++)
2288                 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
2289         WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, adev->gfx.ce_fw_version);
2290
2291         /* ME */
2292         fw_data = (const __le32 *)
2293                 (adev->gfx.me_fw->data +
2294                  le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
2295         fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
2296         WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, 0);
2297         for (i = 0; i < fw_size; i++)
2298                 WREG32_SOC15(GC, 0, mmCP_ME_RAM_DATA, le32_to_cpup(fw_data++));
2299         WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, adev->gfx.me_fw_version);
2300
2301         return 0;
2302 }
2303
2304 static int gfx_v9_0_cp_gfx_start(struct amdgpu_device *adev)
2305 {
2306         struct amdgpu_ring *ring = &adev->gfx.gfx_ring[0];
2307         const struct cs_section_def *sect = NULL;
2308         const struct cs_extent_def *ext = NULL;
2309         int r, i, tmp;
2310
2311         /* init the CP */
2312         WREG32_SOC15(GC, 0, mmCP_MAX_CONTEXT, adev->gfx.config.max_hw_contexts - 1);
2313         WREG32_SOC15(GC, 0, mmCP_DEVICE_ID, 1);
2314
2315         gfx_v9_0_cp_gfx_enable(adev, true);
2316
2317         r = amdgpu_ring_alloc(ring, gfx_v9_0_get_csb_size(adev) + 4 + 3);
2318         if (r) {
2319                 DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r);
2320                 return r;
2321         }
2322
2323         amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2324         amdgpu_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
2325
2326         amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
2327         amdgpu_ring_write(ring, 0x80000000);
2328         amdgpu_ring_write(ring, 0x80000000);
2329
2330         for (sect = gfx9_cs_data; sect->section != NULL; ++sect) {
2331                 for (ext = sect->section; ext->extent != NULL; ++ext) {
2332                         if (sect->id == SECT_CONTEXT) {
2333                                 amdgpu_ring_write(ring,
2334                                        PACKET3(PACKET3_SET_CONTEXT_REG,
2335                                                ext->reg_count));
2336                                 amdgpu_ring_write(ring,
2337                                        ext->reg_index - PACKET3_SET_CONTEXT_REG_START);
2338                                 for (i = 0; i < ext->reg_count; i++)
2339                                         amdgpu_ring_write(ring, ext->extent[i]);
2340                         }
2341                 }
2342         }
2343
2344         amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2345         amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
2346
2347         amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
2348         amdgpu_ring_write(ring, 0);
2349
2350         amdgpu_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
2351         amdgpu_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
2352         amdgpu_ring_write(ring, 0x8000);
2353         amdgpu_ring_write(ring, 0x8000);
2354
2355         amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG,1));
2356         tmp = (PACKET3_SET_UCONFIG_REG_INDEX_TYPE |
2357                 (SOC15_REG_OFFSET(GC, 0, mmVGT_INDEX_TYPE) - PACKET3_SET_UCONFIG_REG_START));
2358         amdgpu_ring_write(ring, tmp);
2359         amdgpu_ring_write(ring, 0);
2360
2361         amdgpu_ring_commit(ring);
2362
2363         return 0;
2364 }
2365
2366 static int gfx_v9_0_cp_gfx_resume(struct amdgpu_device *adev)
2367 {
2368         struct amdgpu_ring *ring;
2369         u32 tmp;
2370         u32 rb_bufsz;
2371         u64 rb_addr, rptr_addr, wptr_gpu_addr;
2372
2373         /* Set the write pointer delay */
2374         WREG32_SOC15(GC, 0, mmCP_RB_WPTR_DELAY, 0);
2375
2376         /* set the RB to use vmid 0 */
2377         WREG32_SOC15(GC, 0, mmCP_RB_VMID, 0);
2378
2379         /* Set ring buffer size */
2380         ring = &adev->gfx.gfx_ring[0];
2381         rb_bufsz = order_base_2(ring->ring_size / 8);
2382         tmp = REG_SET_FIELD(0, CP_RB0_CNTL, RB_BUFSZ, rb_bufsz);
2383         tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, RB_BLKSZ, rb_bufsz - 2);
2384 #ifdef __BIG_ENDIAN
2385         tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, BUF_SWAP, 1);
2386 #endif
2387         WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp);
2388
2389         /* Initialize the ring buffer's write pointers */
2390         ring->wptr = 0;
2391         WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr));
2392         WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr));
2393
2394         /* set the wb address wether it's enabled or not */
2395         rptr_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4);
2396         WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr));
2397         WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & CP_RB_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK);
2398
2399         wptr_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
2400         WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_LO, lower_32_bits(wptr_gpu_addr));
2401         WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_HI, upper_32_bits(wptr_gpu_addr));
2402
2403         mdelay(1);
2404         WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp);
2405
2406         rb_addr = ring->gpu_addr >> 8;
2407         WREG32_SOC15(GC, 0, mmCP_RB0_BASE, rb_addr);
2408         WREG32_SOC15(GC, 0, mmCP_RB0_BASE_HI, upper_32_bits(rb_addr));
2409
2410         tmp = RREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL);
2411         if (ring->use_doorbell) {
2412                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
2413                                     DOORBELL_OFFSET, ring->doorbell_index);
2414                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
2415                                     DOORBELL_EN, 1);
2416         } else {
2417                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, DOORBELL_EN, 0);
2418         }
2419         WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL, tmp);
2420
2421         tmp = REG_SET_FIELD(0, CP_RB_DOORBELL_RANGE_LOWER,
2422                         DOORBELL_RANGE_LOWER, ring->doorbell_index);
2423         WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_LOWER, tmp);
2424
2425         WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_UPPER,
2426                        CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK);
2427
2428
2429         /* start the ring */
2430         gfx_v9_0_cp_gfx_start(adev);
2431         ring->ready = true;
2432
2433         return 0;
2434 }
2435
2436 static void gfx_v9_0_cp_compute_enable(struct amdgpu_device *adev, bool enable)
2437 {
2438         int i;
2439
2440         if (enable) {
2441                 WREG32_SOC15(GC, 0, mmCP_MEC_CNTL, 0);
2442         } else {
2443                 WREG32_SOC15(GC, 0, mmCP_MEC_CNTL,
2444                         (CP_MEC_CNTL__MEC_ME1_HALT_MASK | CP_MEC_CNTL__MEC_ME2_HALT_MASK));
2445                 for (i = 0; i < adev->gfx.num_compute_rings; i++)
2446                         adev->gfx.compute_ring[i].ready = false;
2447                 adev->gfx.kiq.ring.ready = false;
2448         }
2449         udelay(50);
2450 }
2451
2452 static int gfx_v9_0_cp_compute_load_microcode(struct amdgpu_device *adev)
2453 {
2454         const struct gfx_firmware_header_v1_0 *mec_hdr;
2455         const __le32 *fw_data;
2456         unsigned i;
2457         u32 tmp;
2458
2459         if (!adev->gfx.mec_fw)
2460                 return -EINVAL;
2461
2462         gfx_v9_0_cp_compute_enable(adev, false);
2463
2464         mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
2465         amdgpu_ucode_print_gfx_hdr(&mec_hdr->header);
2466
2467         fw_data = (const __le32 *)
2468                 (adev->gfx.mec_fw->data +
2469                  le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
2470         tmp = 0;
2471         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0);
2472         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0);
2473         WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_CNTL, tmp);
2474
2475         WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_LO,
2476                 adev->gfx.mec.mec_fw_gpu_addr & 0xFFFFF000);
2477         WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_HI,
2478                 upper_32_bits(adev->gfx.mec.mec_fw_gpu_addr));
2479
2480         /* MEC1 */
2481         WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR,
2482                          mec_hdr->jt_offset);
2483         for (i = 0; i < mec_hdr->jt_size; i++)
2484                 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_DATA,
2485                         le32_to_cpup(fw_data + mec_hdr->jt_offset + i));
2486
2487         WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR,
2488                         adev->gfx.mec_fw_version);
2489         /* Todo : Loading MEC2 firmware is only necessary if MEC2 should run different microcode than MEC1. */
2490
2491         return 0;
2492 }
2493
2494 /* KIQ functions */
2495 static void gfx_v9_0_kiq_setting(struct amdgpu_ring *ring)
2496 {
2497         uint32_t tmp;
2498         struct amdgpu_device *adev = ring->adev;
2499
2500         /* tell RLC which is KIQ queue */
2501         tmp = RREG32_SOC15(GC, 0, mmRLC_CP_SCHEDULERS);
2502         tmp &= 0xffffff00;
2503         tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue);
2504         WREG32_SOC15(GC, 0, mmRLC_CP_SCHEDULERS, tmp);
2505         tmp |= 0x80;
2506         WREG32_SOC15(GC, 0, mmRLC_CP_SCHEDULERS, tmp);
2507 }
2508
2509 static int gfx_v9_0_kiq_kcq_enable(struct amdgpu_device *adev)
2510 {
2511         struct amdgpu_ring *kiq_ring = &adev->gfx.kiq.ring;
2512         uint32_t scratch, tmp = 0;
2513         uint64_t queue_mask = 0;
2514         int r, i;
2515
2516         for (i = 0; i < AMDGPU_MAX_COMPUTE_QUEUES; ++i) {
2517                 if (!test_bit(i, adev->gfx.mec.queue_bitmap))
2518                         continue;
2519
2520                 /* This situation may be hit in the future if a new HW
2521                  * generation exposes more than 64 queues. If so, the
2522                  * definition of queue_mask needs updating */
2523                 if (WARN_ON(i >= (sizeof(queue_mask)*8))) {
2524                         DRM_ERROR("Invalid KCQ enabled: %d\n", i);
2525                         break;
2526                 }
2527
2528                 queue_mask |= (1ull << i);
2529         }
2530
2531         r = amdgpu_gfx_scratch_get(adev, &scratch);
2532         if (r) {
2533                 DRM_ERROR("Failed to get scratch reg (%d).\n", r);
2534                 return r;
2535         }
2536         WREG32(scratch, 0xCAFEDEAD);
2537
2538         r = amdgpu_ring_alloc(kiq_ring, (7 * adev->gfx.num_compute_rings) + 11);
2539         if (r) {
2540                 DRM_ERROR("Failed to lock KIQ (%d).\n", r);
2541                 amdgpu_gfx_scratch_free(adev, scratch);
2542                 return r;
2543         }
2544
2545         /* set resources */
2546         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_RESOURCES, 6));
2547         amdgpu_ring_write(kiq_ring, PACKET3_SET_RESOURCES_VMID_MASK(0) |
2548                           PACKET3_SET_RESOURCES_QUEUE_TYPE(0)); /* vmid_mask:0 queue_type:0 (KIQ) */
2549         amdgpu_ring_write(kiq_ring, lower_32_bits(queue_mask)); /* queue mask lo */
2550         amdgpu_ring_write(kiq_ring, upper_32_bits(queue_mask)); /* queue mask hi */
2551         amdgpu_ring_write(kiq_ring, 0); /* gws mask lo */
2552         amdgpu_ring_write(kiq_ring, 0); /* gws mask hi */
2553         amdgpu_ring_write(kiq_ring, 0); /* oac mask */
2554         amdgpu_ring_write(kiq_ring, 0); /* gds heap base:0, gds heap size:0 */
2555         for (i = 0; i < adev->gfx.num_compute_rings; i++) {
2556                 struct amdgpu_ring *ring = &adev->gfx.compute_ring[i];
2557                 uint64_t mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj);
2558                 uint64_t wptr_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
2559
2560                 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5));
2561                 /* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/
2562                 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
2563                                   PACKET3_MAP_QUEUES_QUEUE_SEL(0) | /* Queue_Sel */
2564                                   PACKET3_MAP_QUEUES_VMID(0) | /* VMID */
2565                                   PACKET3_MAP_QUEUES_QUEUE(ring->queue) |
2566                                   PACKET3_MAP_QUEUES_PIPE(ring->pipe) |
2567                                   PACKET3_MAP_QUEUES_ME((ring->me == 1 ? 0 : 1)) |
2568                                   PACKET3_MAP_QUEUES_QUEUE_TYPE(0) | /*queue_type: normal compute queue */
2569                                   PACKET3_MAP_QUEUES_ALLOC_FORMAT(0) | /* alloc format: all_on_one_pipe */
2570                                   PACKET3_MAP_QUEUES_ENGINE_SEL(0) | /* engine_sel: compute */
2571                                   PACKET3_MAP_QUEUES_NUM_QUEUES(1)); /* num_queues: must be 1 */
2572                 amdgpu_ring_write(kiq_ring, PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index));
2573                 amdgpu_ring_write(kiq_ring, lower_32_bits(mqd_addr));
2574                 amdgpu_ring_write(kiq_ring, upper_32_bits(mqd_addr));
2575                 amdgpu_ring_write(kiq_ring, lower_32_bits(wptr_addr));
2576                 amdgpu_ring_write(kiq_ring, upper_32_bits(wptr_addr));
2577         }
2578         /* write to scratch for completion */
2579         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
2580         amdgpu_ring_write(kiq_ring, (scratch - PACKET3_SET_UCONFIG_REG_START));
2581         amdgpu_ring_write(kiq_ring, 0xDEADBEEF);
2582         amdgpu_ring_commit(kiq_ring);
2583
2584         for (i = 0; i < adev->usec_timeout; i++) {
2585                 tmp = RREG32(scratch);
2586                 if (tmp == 0xDEADBEEF)
2587                         break;
2588                 DRM_UDELAY(1);
2589         }
2590         if (i >= adev->usec_timeout) {
2591                 DRM_ERROR("KCQ enable failed (scratch(0x%04X)=0x%08X)\n",
2592                           scratch, tmp);
2593                 r = -EINVAL;
2594         }
2595         amdgpu_gfx_scratch_free(adev, scratch);
2596
2597         return r;
2598 }
2599
2600 static int gfx_v9_0_mqd_init(struct amdgpu_ring *ring)
2601 {
2602         struct amdgpu_device *adev = ring->adev;
2603         struct v9_mqd *mqd = ring->mqd_ptr;
2604         uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr;
2605         uint32_t tmp;
2606
2607         mqd->header = 0xC0310800;
2608         mqd->compute_pipelinestat_enable = 0x00000001;
2609         mqd->compute_static_thread_mgmt_se0 = 0xffffffff;
2610         mqd->compute_static_thread_mgmt_se1 = 0xffffffff;
2611         mqd->compute_static_thread_mgmt_se2 = 0xffffffff;
2612         mqd->compute_static_thread_mgmt_se3 = 0xffffffff;
2613         mqd->compute_misc_reserved = 0x00000003;
2614
2615         mqd->dynamic_cu_mask_addr_lo =
2616                 lower_32_bits(ring->mqd_gpu_addr
2617                               + offsetof(struct v9_mqd_allocation, dynamic_cu_mask));
2618         mqd->dynamic_cu_mask_addr_hi =
2619                 upper_32_bits(ring->mqd_gpu_addr
2620                               + offsetof(struct v9_mqd_allocation, dynamic_cu_mask));
2621
2622         eop_base_addr = ring->eop_gpu_addr >> 8;
2623         mqd->cp_hqd_eop_base_addr_lo = eop_base_addr;
2624         mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr);
2625
2626         /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
2627         tmp = RREG32_SOC15(GC, 0, mmCP_HQD_EOP_CONTROL);
2628         tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE,
2629                         (order_base_2(GFX9_MEC_HPD_SIZE / 4) - 1));
2630
2631         mqd->cp_hqd_eop_control = tmp;
2632
2633         /* enable doorbell? */
2634         tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL);
2635
2636         if (ring->use_doorbell) {
2637                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
2638                                     DOORBELL_OFFSET, ring->doorbell_index);
2639                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
2640                                     DOORBELL_EN, 1);
2641                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
2642                                     DOORBELL_SOURCE, 0);
2643                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
2644                                     DOORBELL_HIT, 0);
2645         } else {
2646                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
2647                                          DOORBELL_EN, 0);
2648         }
2649
2650         mqd->cp_hqd_pq_doorbell_control = tmp;
2651
2652         /* disable the queue if it's active */
2653         ring->wptr = 0;
2654         mqd->cp_hqd_dequeue_request = 0;
2655         mqd->cp_hqd_pq_rptr = 0;
2656         mqd->cp_hqd_pq_wptr_lo = 0;
2657         mqd->cp_hqd_pq_wptr_hi = 0;
2658
2659         /* set the pointer to the MQD */
2660         mqd->cp_mqd_base_addr_lo = ring->mqd_gpu_addr & 0xfffffffc;
2661         mqd->cp_mqd_base_addr_hi = upper_32_bits(ring->mqd_gpu_addr);
2662
2663         /* set MQD vmid to 0 */
2664         tmp = RREG32_SOC15(GC, 0, mmCP_MQD_CONTROL);
2665         tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0);
2666         mqd->cp_mqd_control = tmp;
2667
2668         /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
2669         hqd_gpu_addr = ring->gpu_addr >> 8;
2670         mqd->cp_hqd_pq_base_lo = hqd_gpu_addr;
2671         mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
2672
2673         /* set up the HQD, this is similar to CP_RB0_CNTL */
2674         tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_CONTROL);
2675         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE,
2676                             (order_base_2(ring->ring_size / 4) - 1));
2677         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE,
2678                         ((order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1) << 8));
2679 #ifdef __BIG_ENDIAN
2680         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ENDIAN_SWAP, 1);
2681 #endif
2682         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 0);
2683         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ROQ_PQ_IB_FLIP, 0);
2684         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1);
2685         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1);
2686         mqd->cp_hqd_pq_control = tmp;
2687
2688         /* set the wb address whether it's enabled or not */
2689         wb_gpu_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4);
2690         mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc;
2691         mqd->cp_hqd_pq_rptr_report_addr_hi =
2692                 upper_32_bits(wb_gpu_addr) & 0xffff;
2693
2694         /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
2695         wb_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
2696         mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc;
2697         mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
2698
2699         tmp = 0;
2700         /* enable the doorbell if requested */
2701         if (ring->use_doorbell) {
2702                 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL);
2703                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
2704                                 DOORBELL_OFFSET, ring->doorbell_index);
2705
2706                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
2707                                          DOORBELL_EN, 1);
2708                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
2709                                          DOORBELL_SOURCE, 0);
2710                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
2711                                          DOORBELL_HIT, 0);
2712         }
2713
2714         mqd->cp_hqd_pq_doorbell_control = tmp;
2715
2716         /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
2717         ring->wptr = 0;
2718         mqd->cp_hqd_pq_rptr = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_RPTR);
2719
2720         /* set the vmid for the queue */
2721         mqd->cp_hqd_vmid = 0;
2722
2723         tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PERSISTENT_STATE);
2724         tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x53);
2725         mqd->cp_hqd_persistent_state = tmp;
2726
2727         /* set MIN_IB_AVAIL_SIZE */
2728         tmp = RREG32_SOC15(GC, 0, mmCP_HQD_IB_CONTROL);
2729         tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 3);
2730         mqd->cp_hqd_ib_control = tmp;
2731
2732         /* activate the queue */
2733         mqd->cp_hqd_active = 1;
2734
2735         return 0;
2736 }
2737
2738 static int gfx_v9_0_kiq_init_register(struct amdgpu_ring *ring)
2739 {
2740         struct amdgpu_device *adev = ring->adev;
2741         struct v9_mqd *mqd = ring->mqd_ptr;
2742         int j;
2743
2744         /* disable wptr polling */
2745         WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0);
2746
2747         WREG32_SOC15(GC, 0, mmCP_HQD_EOP_BASE_ADDR,
2748                mqd->cp_hqd_eop_base_addr_lo);
2749         WREG32_SOC15(GC, 0, mmCP_HQD_EOP_BASE_ADDR_HI,
2750                mqd->cp_hqd_eop_base_addr_hi);
2751
2752         /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
2753         WREG32_SOC15(GC, 0, mmCP_HQD_EOP_CONTROL,
2754                mqd->cp_hqd_eop_control);
2755
2756         /* enable doorbell? */
2757         WREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL,
2758                mqd->cp_hqd_pq_doorbell_control);
2759
2760         /* disable the queue if it's active */
2761         if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) {
2762                 WREG32_SOC15(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1);
2763                 for (j = 0; j < adev->usec_timeout; j++) {
2764                         if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1))
2765                                 break;
2766                         udelay(1);
2767                 }
2768                 WREG32_SOC15(GC, 0, mmCP_HQD_DEQUEUE_REQUEST,
2769                        mqd->cp_hqd_dequeue_request);
2770                 WREG32_SOC15(GC, 0, mmCP_HQD_PQ_RPTR,
2771                        mqd->cp_hqd_pq_rptr);
2772                 WREG32_SOC15(GC, 0, mmCP_HQD_PQ_WPTR_LO,
2773                        mqd->cp_hqd_pq_wptr_lo);
2774                 WREG32_SOC15(GC, 0, mmCP_HQD_PQ_WPTR_HI,
2775                        mqd->cp_hqd_pq_wptr_hi);
2776         }
2777
2778         /* set the pointer to the MQD */
2779         WREG32_SOC15(GC, 0, mmCP_MQD_BASE_ADDR,
2780                mqd->cp_mqd_base_addr_lo);
2781         WREG32_SOC15(GC, 0, mmCP_MQD_BASE_ADDR_HI,
2782                mqd->cp_mqd_base_addr_hi);
2783
2784         /* set MQD vmid to 0 */
2785         WREG32_SOC15(GC, 0, mmCP_MQD_CONTROL,
2786                mqd->cp_mqd_control);
2787
2788         /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
2789         WREG32_SOC15(GC, 0, mmCP_HQD_PQ_BASE,
2790                mqd->cp_hqd_pq_base_lo);
2791         WREG32_SOC15(GC, 0, mmCP_HQD_PQ_BASE_HI,
2792                mqd->cp_hqd_pq_base_hi);
2793
2794         /* set up the HQD, this is similar to CP_RB0_CNTL */
2795         WREG32_SOC15(GC, 0, mmCP_HQD_PQ_CONTROL,
2796                mqd->cp_hqd_pq_control);
2797
2798         /* set the wb address whether it's enabled or not */
2799         WREG32_SOC15(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR,
2800                                 mqd->cp_hqd_pq_rptr_report_addr_lo);
2801         WREG32_SOC15(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR_HI,
2802                                 mqd->cp_hqd_pq_rptr_report_addr_hi);
2803
2804         /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
2805         WREG32_SOC15(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR,
2806                mqd->cp_hqd_pq_wptr_poll_addr_lo);
2807         WREG32_SOC15(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR_HI,
2808                mqd->cp_hqd_pq_wptr_poll_addr_hi);
2809
2810         /* enable the doorbell if requested */
2811         if (ring->use_doorbell) {
2812                 WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_LOWER,
2813                                         (AMDGPU_DOORBELL64_KIQ *2) << 2);
2814                 WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_UPPER,
2815                                         (AMDGPU_DOORBELL64_USERQUEUE_END * 2) << 2);
2816         }
2817
2818         WREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL,
2819                mqd->cp_hqd_pq_doorbell_control);
2820
2821         /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
2822         WREG32_SOC15(GC, 0, mmCP_HQD_PQ_WPTR_LO,
2823                mqd->cp_hqd_pq_wptr_lo);
2824         WREG32_SOC15(GC, 0, mmCP_HQD_PQ_WPTR_HI,
2825                mqd->cp_hqd_pq_wptr_hi);
2826
2827         /* set the vmid for the queue */
2828         WREG32_SOC15(GC, 0, mmCP_HQD_VMID, mqd->cp_hqd_vmid);
2829
2830         WREG32_SOC15(GC, 0, mmCP_HQD_PERSISTENT_STATE,
2831                mqd->cp_hqd_persistent_state);
2832
2833         /* activate the queue */
2834         WREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE,
2835                mqd->cp_hqd_active);
2836
2837         if (ring->use_doorbell)
2838                 WREG32_FIELD15(GC, 0, CP_PQ_STATUS, DOORBELL_ENABLE, 1);
2839
2840         return 0;
2841 }
2842
2843 static int gfx_v9_0_kiq_fini_register(struct amdgpu_ring *ring)
2844 {
2845         struct amdgpu_device *adev = ring->adev;
2846         int j;
2847
2848         /* disable the queue if it's active */
2849         if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) {
2850
2851                 WREG32_SOC15(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1);
2852
2853                 for (j = 0; j < adev->usec_timeout; j++) {
2854                         if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1))
2855                                 break;
2856                         udelay(1);
2857                 }
2858
2859                 if (j == AMDGPU_MAX_USEC_TIMEOUT) {
2860                         DRM_DEBUG("KIQ dequeue request failed.\n");
2861
2862                         /* Manual disable if dequeue request times out */
2863                         WREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE, 0);
2864                 }
2865
2866                 WREG32_SOC15(GC, 0, mmCP_HQD_DEQUEUE_REQUEST,
2867                       0);
2868         }
2869
2870         WREG32_SOC15(GC, 0, mmCP_HQD_IQ_TIMER, 0);
2871         WREG32_SOC15(GC, 0, mmCP_HQD_IB_CONTROL, 0);
2872         WREG32_SOC15(GC, 0, mmCP_HQD_PERSISTENT_STATE, 0);
2873         WREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0x40000000);
2874         WREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0);
2875         WREG32_SOC15(GC, 0, mmCP_HQD_PQ_RPTR, 0);
2876         WREG32_SOC15(GC, 0, mmCP_HQD_PQ_WPTR_HI, 0);
2877         WREG32_SOC15(GC, 0, mmCP_HQD_PQ_WPTR_LO, 0);
2878
2879         return 0;
2880 }
2881
2882 static int gfx_v9_0_kiq_init_queue(struct amdgpu_ring *ring)
2883 {
2884         struct amdgpu_device *adev = ring->adev;
2885         struct v9_mqd *mqd = ring->mqd_ptr;
2886         int mqd_idx = AMDGPU_MAX_COMPUTE_RINGS;
2887
2888         gfx_v9_0_kiq_setting(ring);
2889
2890         if (adev->in_gpu_reset) { /* for GPU_RESET case */
2891                 /* reset MQD to a clean status */
2892                 if (adev->gfx.mec.mqd_backup[mqd_idx])
2893                         memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation));
2894
2895                 /* reset ring buffer */
2896                 ring->wptr = 0;
2897                 amdgpu_ring_clear_ring(ring);
2898
2899                 mutex_lock(&adev->srbm_mutex);
2900                 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
2901                 gfx_v9_0_kiq_init_register(ring);
2902                 soc15_grbm_select(adev, 0, 0, 0, 0);
2903                 mutex_unlock(&adev->srbm_mutex);
2904         } else {
2905                 memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation));
2906                 ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;
2907                 ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;
2908                 mutex_lock(&adev->srbm_mutex);
2909                 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
2910                 gfx_v9_0_mqd_init(ring);
2911                 gfx_v9_0_kiq_init_register(ring);
2912                 soc15_grbm_select(adev, 0, 0, 0, 0);
2913                 mutex_unlock(&adev->srbm_mutex);
2914
2915                 if (adev->gfx.mec.mqd_backup[mqd_idx])
2916                         memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation));
2917         }
2918
2919         return 0;
2920 }
2921
2922 static int gfx_v9_0_kcq_init_queue(struct amdgpu_ring *ring)
2923 {
2924         struct amdgpu_device *adev = ring->adev;
2925         struct v9_mqd *mqd = ring->mqd_ptr;
2926         int mqd_idx = ring - &adev->gfx.compute_ring[0];
2927
2928         if (!adev->in_gpu_reset && !adev->gfx.in_suspend) {
2929                 memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation));
2930                 ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;
2931                 ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;
2932                 mutex_lock(&adev->srbm_mutex);
2933                 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
2934                 gfx_v9_0_mqd_init(ring);
2935                 soc15_grbm_select(adev, 0, 0, 0, 0);
2936                 mutex_unlock(&adev->srbm_mutex);
2937
2938                 if (adev->gfx.mec.mqd_backup[mqd_idx])
2939                         memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation));
2940         } else if (adev->in_gpu_reset) { /* for GPU_RESET case */
2941                 /* reset MQD to a clean status */
2942                 if (adev->gfx.mec.mqd_backup[mqd_idx])
2943                         memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation));
2944
2945                 /* reset ring buffer */
2946                 ring->wptr = 0;
2947                 amdgpu_ring_clear_ring(ring);
2948         } else {
2949                 amdgpu_ring_clear_ring(ring);
2950         }
2951
2952         return 0;
2953 }
2954
2955 static int gfx_v9_0_kiq_resume(struct amdgpu_device *adev)
2956 {
2957         struct amdgpu_ring *ring = NULL;
2958         int r = 0, i;
2959
2960         gfx_v9_0_cp_compute_enable(adev, true);
2961
2962         ring = &adev->gfx.kiq.ring;
2963
2964         r = amdgpu_bo_reserve(ring->mqd_obj, false);
2965         if (unlikely(r != 0))
2966                 goto done;
2967
2968         r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
2969         if (!r) {
2970                 r = gfx_v9_0_kiq_init_queue(ring);
2971                 amdgpu_bo_kunmap(ring->mqd_obj);
2972                 ring->mqd_ptr = NULL;
2973         }
2974         amdgpu_bo_unreserve(ring->mqd_obj);
2975         if (r)
2976                 goto done;
2977
2978         for (i = 0; i < adev->gfx.num_compute_rings; i++) {
2979                 ring = &adev->gfx.compute_ring[i];
2980
2981                 r = amdgpu_bo_reserve(ring->mqd_obj, false);
2982                 if (unlikely(r != 0))
2983                         goto done;
2984                 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
2985                 if (!r) {
2986                         r = gfx_v9_0_kcq_init_queue(ring);
2987                         amdgpu_bo_kunmap(ring->mqd_obj);
2988                         ring->mqd_ptr = NULL;
2989                 }
2990                 amdgpu_bo_unreserve(ring->mqd_obj);
2991                 if (r)
2992                         goto done;
2993         }
2994
2995         r = gfx_v9_0_kiq_kcq_enable(adev);
2996 done:
2997         return r;
2998 }
2999
3000 static int gfx_v9_0_cp_resume(struct amdgpu_device *adev)
3001 {
3002         int r, i;
3003         struct amdgpu_ring *ring;
3004
3005         if (!(adev->flags & AMD_IS_APU))
3006                 gfx_v9_0_enable_gui_idle_interrupt(adev, false);
3007
3008         if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {
3009                 /* legacy firmware loading */
3010                 r = gfx_v9_0_cp_gfx_load_microcode(adev);
3011                 if (r)
3012                         return r;
3013
3014                 r = gfx_v9_0_cp_compute_load_microcode(adev);
3015                 if (r)
3016                         return r;
3017         }
3018
3019         r = gfx_v9_0_cp_gfx_resume(adev);
3020         if (r)
3021                 return r;
3022
3023         r = gfx_v9_0_kiq_resume(adev);
3024         if (r)
3025                 return r;
3026
3027         ring = &adev->gfx.gfx_ring[0];
3028         r = amdgpu_ring_test_ring(ring);
3029         if (r) {
3030                 ring->ready = false;
3031                 return r;
3032         }
3033
3034         ring = &adev->gfx.kiq.ring;
3035         ring->ready = true;
3036         r = amdgpu_ring_test_ring(ring);
3037         if (r)
3038                 ring->ready = false;
3039
3040         for (i = 0; i < adev->gfx.num_compute_rings; i++) {
3041                 ring = &adev->gfx.compute_ring[i];
3042
3043                 ring->ready = true;
3044                 r = amdgpu_ring_test_ring(ring);
3045                 if (r)
3046                         ring->ready = false;
3047         }
3048
3049         gfx_v9_0_enable_gui_idle_interrupt(adev, true);
3050
3051         return 0;
3052 }
3053
3054 static void gfx_v9_0_cp_enable(struct amdgpu_device *adev, bool enable)
3055 {
3056         gfx_v9_0_cp_gfx_enable(adev, enable);
3057         gfx_v9_0_cp_compute_enable(adev, enable);
3058 }
3059
3060 static int gfx_v9_0_hw_init(void *handle)
3061 {
3062         int r;
3063         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3064
3065         gfx_v9_0_init_golden_registers(adev);
3066
3067         gfx_v9_0_gpu_init(adev);
3068
3069         r = gfx_v9_0_rlc_resume(adev);
3070         if (r)
3071                 return r;
3072
3073         r = gfx_v9_0_cp_resume(adev);
3074         if (r)
3075                 return r;
3076
3077         r = gfx_v9_0_ngg_en(adev);
3078         if (r)
3079                 return r;
3080
3081         return r;
3082 }
3083
3084 static int gfx_v9_0_kcq_disable(struct amdgpu_ring *kiq_ring,struct amdgpu_ring *ring)
3085 {
3086         struct amdgpu_device *adev = kiq_ring->adev;
3087         uint32_t scratch, tmp = 0;
3088         int r, i;
3089
3090         r = amdgpu_gfx_scratch_get(adev, &scratch);
3091         if (r) {
3092                 DRM_ERROR("Failed to get scratch reg (%d).\n", r);
3093                 return r;
3094         }
3095         WREG32(scratch, 0xCAFEDEAD);
3096
3097         r = amdgpu_ring_alloc(kiq_ring, 10);
3098         if (r) {
3099                 DRM_ERROR("Failed to lock KIQ (%d).\n", r);
3100                 amdgpu_gfx_scratch_free(adev, scratch);
3101                 return r;
3102         }
3103
3104         /* unmap queues */
3105         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4));
3106         amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
3107                                                 PACKET3_UNMAP_QUEUES_ACTION(1) | /* RESET_QUEUES */
3108                                                 PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) |
3109                                                 PACKET3_UNMAP_QUEUES_ENGINE_SEL(0) |
3110                                                 PACKET3_UNMAP_QUEUES_NUM_QUEUES(1));
3111         amdgpu_ring_write(kiq_ring, PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index));
3112         amdgpu_ring_write(kiq_ring, 0);
3113         amdgpu_ring_write(kiq_ring, 0);
3114         amdgpu_ring_write(kiq_ring, 0);
3115         /* write to scratch for completion */
3116         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3117         amdgpu_ring_write(kiq_ring, (scratch - PACKET3_SET_UCONFIG_REG_START));
3118         amdgpu_ring_write(kiq_ring, 0xDEADBEEF);
3119         amdgpu_ring_commit(kiq_ring);
3120
3121         for (i = 0; i < adev->usec_timeout; i++) {
3122                 tmp = RREG32(scratch);
3123                 if (tmp == 0xDEADBEEF)
3124                         break;
3125                 DRM_UDELAY(1);
3126         }
3127         if (i >= adev->usec_timeout) {
3128                 DRM_ERROR("KCQ disabled failed (scratch(0x%04X)=0x%08X)\n", scratch, tmp);
3129                 r = -EINVAL;
3130         }
3131         amdgpu_gfx_scratch_free(adev, scratch);
3132         return r;
3133 }
3134
3135 static int gfx_v9_0_hw_fini(void *handle)
3136 {
3137         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3138         int i;
3139
3140         amdgpu_device_ip_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_GFX,
3141                                                AMD_PG_STATE_UNGATE);
3142
3143         amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
3144         amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
3145
3146         /* disable KCQ to avoid CPC touch memory not valid anymore */
3147         for (i = 0; i < adev->gfx.num_compute_rings; i++)
3148                 gfx_v9_0_kcq_disable(&adev->gfx.kiq.ring, &adev->gfx.compute_ring[i]);
3149
3150         if (amdgpu_sriov_vf(adev)) {
3151                 gfx_v9_0_cp_gfx_enable(adev, false);
3152                 /* must disable polling for SRIOV when hw finished, otherwise
3153                  * CPC engine may still keep fetching WB address which is already
3154                  * invalid after sw finished and trigger DMAR reading error in
3155                  * hypervisor side.
3156                  */
3157                 WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0);
3158                 return 0;
3159         }
3160
3161         /* Use deinitialize sequence from CAIL when unbinding device from driver,
3162          * otherwise KIQ is hanging when binding back
3163          */
3164         if (!adev->in_gpu_reset && !adev->gfx.in_suspend) {
3165                 mutex_lock(&adev->srbm_mutex);
3166                 soc15_grbm_select(adev, adev->gfx.kiq.ring.me,
3167                                 adev->gfx.kiq.ring.pipe,
3168                                 adev->gfx.kiq.ring.queue, 0);
3169                 gfx_v9_0_kiq_fini_register(&adev->gfx.kiq.ring);
3170                 soc15_grbm_select(adev, 0, 0, 0, 0);
3171                 mutex_unlock(&adev->srbm_mutex);
3172         }
3173
3174         gfx_v9_0_cp_enable(adev, false);
3175         gfx_v9_0_rlc_stop(adev);
3176
3177         return 0;
3178 }
3179
3180 static int gfx_v9_0_suspend(void *handle)
3181 {
3182         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3183
3184         adev->gfx.in_suspend = true;
3185         return gfx_v9_0_hw_fini(adev);
3186 }
3187
3188 static int gfx_v9_0_resume(void *handle)
3189 {
3190         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3191         int r;
3192
3193         r = gfx_v9_0_hw_init(adev);
3194         adev->gfx.in_suspend = false;
3195         return r;
3196 }
3197
3198 static bool gfx_v9_0_is_idle(void *handle)
3199 {
3200         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3201
3202         if (REG_GET_FIELD(RREG32_SOC15(GC, 0, mmGRBM_STATUS),
3203                                 GRBM_STATUS, GUI_ACTIVE))
3204                 return false;
3205         else
3206                 return true;
3207 }
3208
3209 static int gfx_v9_0_wait_for_idle(void *handle)
3210 {
3211         unsigned i;
3212         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3213
3214         for (i = 0; i < adev->usec_timeout; i++) {
3215                 if (gfx_v9_0_is_idle(handle))
3216                         return 0;
3217                 udelay(1);
3218         }
3219         return -ETIMEDOUT;
3220 }
3221
3222 static int gfx_v9_0_soft_reset(void *handle)
3223 {
3224         u32 grbm_soft_reset = 0;
3225         u32 tmp;
3226         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3227
3228         /* GRBM_STATUS */
3229         tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS);
3230         if (tmp & (GRBM_STATUS__PA_BUSY_MASK | GRBM_STATUS__SC_BUSY_MASK |
3231                    GRBM_STATUS__BCI_BUSY_MASK | GRBM_STATUS__SX_BUSY_MASK |
3232                    GRBM_STATUS__TA_BUSY_MASK | GRBM_STATUS__VGT_BUSY_MASK |
3233                    GRBM_STATUS__DB_BUSY_MASK | GRBM_STATUS__CB_BUSY_MASK |
3234                    GRBM_STATUS__GDS_BUSY_MASK | GRBM_STATUS__SPI_BUSY_MASK |
3235                    GRBM_STATUS__IA_BUSY_MASK | GRBM_STATUS__IA_BUSY_NO_DMA_MASK)) {
3236                 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
3237                                                 GRBM_SOFT_RESET, SOFT_RESET_CP, 1);
3238                 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
3239                                                 GRBM_SOFT_RESET, SOFT_RESET_GFX, 1);
3240         }
3241
3242         if (tmp & (GRBM_STATUS__CP_BUSY_MASK | GRBM_STATUS__CP_COHERENCY_BUSY_MASK)) {
3243                 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
3244                                                 GRBM_SOFT_RESET, SOFT_RESET_CP, 1);
3245         }
3246
3247         /* GRBM_STATUS2 */
3248         tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS2);
3249         if (REG_GET_FIELD(tmp, GRBM_STATUS2, RLC_BUSY))
3250                 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
3251                                                 GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
3252
3253
3254         if (grbm_soft_reset) {
3255                 /* stop the rlc */
3256                 gfx_v9_0_rlc_stop(adev);
3257
3258                 /* Disable GFX parsing/prefetching */
3259                 gfx_v9_0_cp_gfx_enable(adev, false);
3260
3261                 /* Disable MEC parsing/prefetching */
3262                 gfx_v9_0_cp_compute_enable(adev, false);
3263
3264                 if (grbm_soft_reset) {
3265                         tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
3266                         tmp |= grbm_soft_reset;
3267                         dev_info(adev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3268                         WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp);
3269                         tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
3270
3271                         udelay(50);
3272
3273                         tmp &= ~grbm_soft_reset;
3274                         WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp);
3275                         tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
3276                 }
3277
3278                 /* Wait a little for things to settle down */
3279                 udelay(50);
3280         }
3281         return 0;
3282 }
3283
3284 static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev)
3285 {
3286         uint64_t clock;
3287
3288         mutex_lock(&adev->gfx.gpu_clock_mutex);
3289         WREG32_SOC15(GC, 0, mmRLC_CAPTURE_GPU_CLOCK_COUNT, 1);
3290         clock = (uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_LSB) |
3291                 ((uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
3292         mutex_unlock(&adev->gfx.gpu_clock_mutex);
3293         return clock;
3294 }
3295
3296 static void gfx_v9_0_ring_emit_gds_switch(struct amdgpu_ring *ring,
3297                                           uint32_t vmid,
3298                                           uint32_t gds_base, uint32_t gds_size,
3299                                           uint32_t gws_base, uint32_t gws_size,
3300                                           uint32_t oa_base, uint32_t oa_size)
3301 {
3302         struct amdgpu_device *adev = ring->adev;
3303
3304         gds_base = gds_base >> AMDGPU_GDS_SHIFT;
3305         gds_size = gds_size >> AMDGPU_GDS_SHIFT;
3306
3307         gws_base = gws_base >> AMDGPU_GWS_SHIFT;
3308         gws_size = gws_size >> AMDGPU_GWS_SHIFT;
3309
3310         oa_base = oa_base >> AMDGPU_OA_SHIFT;
3311         oa_size = oa_size >> AMDGPU_OA_SHIFT;
3312
3313         /* GDS Base */
3314         gfx_v9_0_write_data_to_reg(ring, 0, false,
3315                                    SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_BASE) + 2 * vmid,
3316                                    gds_base);
3317
3318         /* GDS Size */
3319         gfx_v9_0_write_data_to_reg(ring, 0, false,
3320                                    SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_SIZE) + 2 * vmid,
3321                                    gds_size);
3322
3323         /* GWS */
3324         gfx_v9_0_write_data_to_reg(ring, 0, false,
3325                                    SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID0) + vmid,
3326                                    gws_size << GDS_GWS_VMID0__SIZE__SHIFT | gws_base);
3327
3328         /* OA */
3329         gfx_v9_0_write_data_to_reg(ring, 0, false,
3330                                    SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID0) + vmid,
3331                                    (1 << (oa_size + oa_base)) - (1 << oa_base));
3332 }
3333
3334 static int gfx_v9_0_early_init(void *handle)
3335 {
3336         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3337
3338         adev->gfx.num_gfx_rings = GFX9_NUM_GFX_RINGS;
3339         adev->gfx.num_compute_rings = AMDGPU_MAX_COMPUTE_RINGS;
3340         gfx_v9_0_set_ring_funcs(adev);
3341         gfx_v9_0_set_irq_funcs(adev);
3342         gfx_v9_0_set_gds_init(adev);
3343         gfx_v9_0_set_rlc_funcs(adev);
3344
3345         return 0;
3346 }
3347
3348 static int gfx_v9_0_late_init(void *handle)
3349 {
3350         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3351         int r;
3352
3353         r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0);
3354         if (r)
3355                 return r;
3356
3357         r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0);
3358         if (r)
3359                 return r;
3360
3361         r = amdgpu_device_ip_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_GFX,
3362                                                    AMD_PG_STATE_GATE);
3363         if (r)
3364                 return r;
3365
3366         return 0;
3367 }
3368
3369 static void gfx_v9_0_enter_rlc_safe_mode(struct amdgpu_device *adev)
3370 {
3371         uint32_t rlc_setting, data;
3372         unsigned i;
3373
3374         if (adev->gfx.rlc.in_safe_mode)
3375                 return;
3376
3377         /* if RLC is not enabled, do nothing */
3378         rlc_setting = RREG32_SOC15(GC, 0, mmRLC_CNTL);
3379         if (!(rlc_setting & RLC_CNTL__RLC_ENABLE_F32_MASK))
3380                 return;
3381
3382         if (adev->cg_flags &
3383             (AMD_CG_SUPPORT_GFX_CGCG | AMD_CG_SUPPORT_GFX_MGCG |
3384              AMD_CG_SUPPORT_GFX_3D_CGCG)) {
3385                 data = RLC_SAFE_MODE__CMD_MASK;
3386                 data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT);
3387                 WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data);
3388
3389                 /* wait for RLC_SAFE_MODE */
3390                 for (i = 0; i < adev->usec_timeout; i++) {
3391                         if (!REG_GET_FIELD(SOC15_REG_OFFSET(GC, 0, mmRLC_SAFE_MODE), RLC_SAFE_MODE, CMD))
3392                                 break;
3393                         udelay(1);
3394                 }
3395                 adev->gfx.rlc.in_safe_mode = true;
3396         }
3397 }
3398
3399 static void gfx_v9_0_exit_rlc_safe_mode(struct amdgpu_device *adev)
3400 {
3401         uint32_t rlc_setting, data;
3402
3403         if (!adev->gfx.rlc.in_safe_mode)
3404                 return;
3405
3406         /* if RLC is not enabled, do nothing */
3407         rlc_setting = RREG32_SOC15(GC, 0, mmRLC_CNTL);
3408         if (!(rlc_setting & RLC_CNTL__RLC_ENABLE_F32_MASK))
3409                 return;
3410
3411         if (adev->cg_flags &
3412             (AMD_CG_SUPPORT_GFX_CGCG | AMD_CG_SUPPORT_GFX_MGCG)) {
3413                 /*
3414                  * Try to exit safe mode only if it is already in safe
3415                  * mode.
3416                  */
3417                 data = RLC_SAFE_MODE__CMD_MASK;
3418                 WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data);
3419                 adev->gfx.rlc.in_safe_mode = false;
3420         }
3421 }
3422
3423 static void gfx_v9_0_update_gfx_cg_power_gating(struct amdgpu_device *adev,
3424                                                 bool enable)
3425 {
3426         gfx_v9_0_enter_rlc_safe_mode(adev);
3427
3428         if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_PG) && enable) {
3429                 gfx_v9_0_enable_gfx_cg_power_gating(adev, true);
3430                 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE)
3431                         gfx_v9_0_enable_gfx_pipeline_powergating(adev, true);
3432         } else {
3433                 gfx_v9_0_enable_gfx_cg_power_gating(adev, false);
3434                 gfx_v9_0_enable_gfx_pipeline_powergating(adev, false);
3435         }
3436
3437         gfx_v9_0_exit_rlc_safe_mode(adev);
3438 }
3439
3440 static void gfx_v9_0_update_gfx_mg_power_gating(struct amdgpu_device *adev,
3441                                                 bool enable)
3442 {
3443         /* TODO: double check if we need to perform under safe mode */
3444         /* gfx_v9_0_enter_rlc_safe_mode(adev); */
3445
3446         if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG) && enable)
3447                 gfx_v9_0_enable_gfx_static_mg_power_gating(adev, true);
3448         else
3449                 gfx_v9_0_enable_gfx_static_mg_power_gating(adev, false);
3450
3451         if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG) && enable)
3452                 gfx_v9_0_enable_gfx_dynamic_mg_power_gating(adev, true);
3453         else
3454                 gfx_v9_0_enable_gfx_dynamic_mg_power_gating(adev, false);
3455
3456         /* gfx_v9_0_exit_rlc_safe_mode(adev); */
3457 }
3458
3459 static void gfx_v9_0_update_medium_grain_clock_gating(struct amdgpu_device *adev,
3460                                                       bool enable)
3461 {
3462         uint32_t data, def;
3463
3464         /* It is disabled by HW by default */
3465         if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG)) {
3466                 /* 1 - RLC_CGTT_MGCG_OVERRIDE */
3467                 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
3468                 data &= ~(RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK |
3469                           RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
3470                           RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK |
3471                           RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK);
3472
3473                 /* only for Vega10 & Raven1 */
3474                 data |= RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK;
3475
3476                 if (def != data)
3477                         WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
3478
3479                 /* MGLS is a global flag to control all MGLS in GFX */
3480                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) {
3481                         /* 2 - RLC memory Light sleep */
3482                         if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS) {
3483                                 def = data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL);
3484                                 data |= RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK;
3485                                 if (def != data)
3486                                         WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data);
3487                         }
3488                         /* 3 - CP memory Light sleep */
3489                         if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) {
3490                                 def = data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL);
3491                                 data |= CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK;
3492                                 if (def != data)
3493                                         WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data);
3494                         }
3495                 }
3496         } else {
3497                 /* 1 - MGCG_OVERRIDE */
3498                 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
3499                 data |= (RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK |
3500                          RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK |
3501                          RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
3502                          RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK |
3503                          RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK);
3504                 if (def != data)
3505                         WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
3506
3507                 /* 2 - disable MGLS in RLC */
3508                 data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL);
3509                 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) {
3510                         data &= ~RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK;
3511                         WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data);
3512                 }
3513
3514                 /* 3 - disable MGLS in CP */
3515                 data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL);
3516                 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) {
3517                         data &= ~CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK;
3518                         WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data);
3519                 }
3520         }
3521 }
3522
3523 static void gfx_v9_0_update_3d_clock_gating(struct amdgpu_device *adev,
3524                                            bool enable)
3525 {
3526         uint32_t data, def;
3527
3528         adev->gfx.rlc.funcs->enter_safe_mode(adev);
3529
3530         /* Enable 3D CGCG/CGLS */
3531         if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG)) {
3532                 /* write cmd to clear cgcg/cgls ov */
3533                 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
3534                 /* unset CGCG override */
3535                 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_GFX3D_CG_OVERRIDE_MASK;
3536                 /* update CGCG and CGLS override bits */
3537                 if (def != data)
3538                         WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
3539                 /* enable 3Dcgcg FSM(0x0020003f) */
3540                 def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D);
3541                 data = (0x2000 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
3542                         RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK;
3543                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS)
3544                         data |= (0x000F << RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
3545                                 RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK;
3546                 if (def != data)
3547                         WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data);
3548
3549                 /* set IDLE_POLL_COUNT(0x00900100) */
3550                 def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL);
3551                 data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) |
3552                         (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
3553                 if (def != data)
3554                         WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data);
3555         } else {
3556                 /* Disable CGCG/CGLS */
3557                 def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D);
3558                 /* disable cgcg, cgls should be disabled */
3559                 data &= ~(RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK |
3560                           RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK);
3561                 /* disable cgcg and cgls in FSM */
3562                 if (def != data)
3563                         WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data);
3564         }
3565
3566         adev->gfx.rlc.funcs->exit_safe_mode(adev);
3567 }
3568
3569 static void gfx_v9_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev,
3570                                                       bool enable)
3571 {
3572         uint32_t def, data;
3573
3574         adev->gfx.rlc.funcs->enter_safe_mode(adev);
3575
3576         if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)) {
3577                 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
3578                 /* unset CGCG override */
3579                 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGCG_OVERRIDE_MASK;
3580                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
3581                         data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK;
3582                 else
3583                         data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK;
3584                 /* update CGCG and CGLS override bits */
3585                 if (def != data)
3586                         WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
3587
3588                 /* enable cgcg FSM(0x0020003F) */
3589                 def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL);
3590                 data = (0x2000 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
3591                         RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
3592                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
3593                         data |= (0x000F << RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
3594                                 RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;
3595                 if (def != data)
3596                         WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data);
3597
3598                 /* set IDLE_POLL_COUNT(0x00900100) */
3599                 def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL);
3600                 data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) |
3601                         (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
3602                 if (def != data)
3603                         WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data);
3604         } else {
3605                 def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL);
3606                 /* reset CGCG/CGLS bits */
3607                 data &= ~(RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK | RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK);
3608                 /* disable cgcg and cgls in FSM */
3609                 if (def != data)
3610                         WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data);
3611         }
3612
3613         adev->gfx.rlc.funcs->exit_safe_mode(adev);
3614 }
3615
3616 static int gfx_v9_0_update_gfx_clock_gating(struct amdgpu_device *adev,
3617                                             bool enable)
3618 {
3619         if (enable) {
3620                 /* CGCG/CGLS should be enabled after MGCG/MGLS
3621                  * ===  MGCG + MGLS ===
3622                  */
3623                 gfx_v9_0_update_medium_grain_clock_gating(adev, enable);
3624                 /* ===  CGCG /CGLS for GFX 3D Only === */
3625                 gfx_v9_0_update_3d_clock_gating(adev, enable);
3626                 /* ===  CGCG + CGLS === */
3627                 gfx_v9_0_update_coarse_grain_clock_gating(adev, enable);
3628         } else {
3629                 /* CGCG/CGLS should be disabled before MGCG/MGLS
3630                  * ===  CGCG + CGLS ===
3631                  */
3632                 gfx_v9_0_update_coarse_grain_clock_gating(adev, enable);
3633                 /* ===  CGCG /CGLS for GFX 3D Only === */
3634                 gfx_v9_0_update_3d_clock_gating(adev, enable);
3635                 /* ===  MGCG + MGLS === */
3636                 gfx_v9_0_update_medium_grain_clock_gating(adev, enable);
3637         }
3638         return 0;
3639 }
3640
3641 static const struct amdgpu_rlc_funcs gfx_v9_0_rlc_funcs = {
3642         .enter_safe_mode = gfx_v9_0_enter_rlc_safe_mode,
3643         .exit_safe_mode = gfx_v9_0_exit_rlc_safe_mode
3644 };
3645
3646 static int gfx_v9_0_set_powergating_state(void *handle,
3647                                           enum amd_powergating_state state)
3648 {
3649         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3650         bool enable = (state == AMD_PG_STATE_GATE) ? true : false;
3651
3652         switch (adev->asic_type) {
3653         case CHIP_RAVEN:
3654                 if (adev->pg_flags & AMD_PG_SUPPORT_RLC_SMU_HS) {
3655                         gfx_v9_0_enable_sck_slow_down_on_power_up(adev, true);
3656                         gfx_v9_0_enable_sck_slow_down_on_power_down(adev, true);
3657                 } else {
3658                         gfx_v9_0_enable_sck_slow_down_on_power_up(adev, false);
3659                         gfx_v9_0_enable_sck_slow_down_on_power_down(adev, false);
3660                 }
3661
3662                 if (adev->pg_flags & AMD_PG_SUPPORT_CP)
3663                         gfx_v9_0_enable_cp_power_gating(adev, true);
3664                 else
3665                         gfx_v9_0_enable_cp_power_gating(adev, false);
3666
3667                 /* update gfx cgpg state */
3668                 gfx_v9_0_update_gfx_cg_power_gating(adev, enable);
3669
3670                 /* update mgcg state */
3671                 gfx_v9_0_update_gfx_mg_power_gating(adev, enable);
3672                 break;
3673         default:
3674                 break;
3675         }
3676
3677         return 0;
3678 }
3679
3680 static int gfx_v9_0_set_clockgating_state(void *handle,
3681                                           enum amd_clockgating_state state)
3682 {
3683         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3684
3685         if (amdgpu_sriov_vf(adev))
3686                 return 0;
3687
3688         switch (adev->asic_type) {
3689         case CHIP_VEGA10:
3690         case CHIP_VEGA12:
3691         case CHIP_RAVEN:
3692                 gfx_v9_0_update_gfx_clock_gating(adev,
3693                                                  state == AMD_CG_STATE_GATE ? true : false);
3694                 break;
3695         default:
3696                 break;
3697         }
3698         return 0;
3699 }
3700
3701 static void gfx_v9_0_get_clockgating_state(void *handle, u32 *flags)
3702 {
3703         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3704         int data;
3705
3706         if (amdgpu_sriov_vf(adev))
3707                 *flags = 0;
3708
3709         /* AMD_CG_SUPPORT_GFX_MGCG */
3710         data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
3711         if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK))
3712                 *flags |= AMD_CG_SUPPORT_GFX_MGCG;
3713
3714         /* AMD_CG_SUPPORT_GFX_CGCG */
3715         data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL);
3716         if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK)
3717                 *flags |= AMD_CG_SUPPORT_GFX_CGCG;
3718
3719         /* AMD_CG_SUPPORT_GFX_CGLS */
3720         if (data & RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK)
3721                 *flags |= AMD_CG_SUPPORT_GFX_CGLS;
3722
3723         /* AMD_CG_SUPPORT_GFX_RLC_LS */
3724         data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL);
3725         if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK)
3726                 *flags |= AMD_CG_SUPPORT_GFX_RLC_LS | AMD_CG_SUPPORT_GFX_MGLS;
3727
3728         /* AMD_CG_SUPPORT_GFX_CP_LS */
3729         data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL);
3730         if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK)
3731                 *flags |= AMD_CG_SUPPORT_GFX_CP_LS | AMD_CG_SUPPORT_GFX_MGLS;
3732
3733         /* AMD_CG_SUPPORT_GFX_3D_CGCG */
3734         data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D);
3735         if (data & RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK)
3736                 *flags |= AMD_CG_SUPPORT_GFX_3D_CGCG;
3737
3738         /* AMD_CG_SUPPORT_GFX_3D_CGLS */
3739         if (data & RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK)
3740                 *flags |= AMD_CG_SUPPORT_GFX_3D_CGLS;
3741 }
3742
3743 static u64 gfx_v9_0_ring_get_rptr_gfx(struct amdgpu_ring *ring)
3744 {
3745         return ring->adev->wb.wb[ring->rptr_offs]; /* gfx9 is 32bit rptr*/
3746 }
3747
3748 static u64 gfx_v9_0_ring_get_wptr_gfx(struct amdgpu_ring *ring)
3749 {
3750         struct amdgpu_device *adev = ring->adev;
3751         u64 wptr;
3752
3753         /* XXX check if swapping is necessary on BE */
3754         if (ring->use_doorbell) {
3755                 wptr = atomic64_read((atomic64_t *)&adev->wb.wb[ring->wptr_offs]);
3756         } else {
3757                 wptr = RREG32_SOC15(GC, 0, mmCP_RB0_WPTR);
3758                 wptr += (u64)RREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI) << 32;
3759         }
3760
3761         return wptr;
3762 }
3763
3764 static void gfx_v9_0_ring_set_wptr_gfx(struct amdgpu_ring *ring)
3765 {
3766         struct amdgpu_device *adev = ring->adev;
3767
3768         if (ring->use_doorbell) {
3769                 /* XXX check if swapping is necessary on BE */
3770                 atomic64_set((atomic64_t*)&adev->wb.wb[ring->wptr_offs], ring->wptr);
3771                 WDOORBELL64(ring->doorbell_index, ring->wptr);
3772         } else {
3773                 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr));
3774                 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr));
3775         }
3776 }
3777
3778 static void gfx_v9_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
3779 {
3780         struct amdgpu_device *adev = ring->adev;
3781         u32 ref_and_mask, reg_mem_engine;
3782         const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio_funcs->hdp_flush_reg;
3783
3784         if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
3785                 switch (ring->me) {
3786                 case 1:
3787                         ref_and_mask = nbio_hf_reg->ref_and_mask_cp2 << ring->pipe;
3788                         break;
3789                 case 2:
3790                         ref_and_mask = nbio_hf_reg->ref_and_mask_cp6 << ring->pipe;
3791                         break;
3792                 default:
3793                         return;
3794                 }
3795                 reg_mem_engine = 0;
3796         } else {
3797                 ref_and_mask = nbio_hf_reg->ref_and_mask_cp0;
3798                 reg_mem_engine = 1; /* pfp */
3799         }
3800
3801         gfx_v9_0_wait_reg_mem(ring, reg_mem_engine, 0, 1,
3802                               adev->nbio_funcs->get_hdp_flush_req_offset(adev),
3803                               adev->nbio_funcs->get_hdp_flush_done_offset(adev),
3804                               ref_and_mask, ref_and_mask, 0x20);
3805 }
3806
3807 static void gfx_v9_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
3808                                       struct amdgpu_ib *ib,
3809                                       unsigned vmid, bool ctx_switch)
3810 {
3811         u32 header, control = 0;
3812
3813         if (ib->flags & AMDGPU_IB_FLAG_CE)
3814                 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
3815         else
3816                 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
3817
3818         control |= ib->length_dw | (vmid << 24);
3819
3820         if (amdgpu_sriov_vf(ring->adev) && (ib->flags & AMDGPU_IB_FLAG_PREEMPT)) {
3821                 control |= INDIRECT_BUFFER_PRE_ENB(1);
3822
3823                 if (!(ib->flags & AMDGPU_IB_FLAG_CE))
3824                         gfx_v9_0_ring_emit_de_meta(ring);
3825         }
3826
3827         amdgpu_ring_write(ring, header);
3828         BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
3829         amdgpu_ring_write(ring,
3830 #ifdef __BIG_ENDIAN
3831                 (2 << 0) |
3832 #endif
3833                 lower_32_bits(ib->gpu_addr));
3834         amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
3835         amdgpu_ring_write(ring, control);
3836 }
3837
3838 static void gfx_v9_0_ring_emit_ib_compute(struct amdgpu_ring *ring,
3839                                           struct amdgpu_ib *ib,
3840                                           unsigned vmid, bool ctx_switch)
3841 {
3842         u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24);
3843
3844         amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
3845         BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
3846         amdgpu_ring_write(ring,
3847 #ifdef __BIG_ENDIAN
3848                                 (2 << 0) |
3849 #endif
3850                                 lower_32_bits(ib->gpu_addr));
3851         amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
3852         amdgpu_ring_write(ring, control);
3853 }
3854
3855 static void gfx_v9_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
3856                                      u64 seq, unsigned flags)
3857 {
3858         bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
3859         bool int_sel = flags & AMDGPU_FENCE_FLAG_INT;
3860         bool writeback = flags & AMDGPU_FENCE_FLAG_TC_WB_ONLY;
3861
3862         /* RELEASE_MEM - flush caches, send int */
3863         amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 6));
3864         amdgpu_ring_write(ring, ((writeback ? (EOP_TC_WB_ACTION_EN |
3865                                                EOP_TC_NC_ACTION_EN) :
3866                                               (EOP_TCL1_ACTION_EN |
3867                                                EOP_TC_ACTION_EN |
3868                                                EOP_TC_WB_ACTION_EN |
3869                                                EOP_TC_MD_ACTION_EN)) |
3870                                  EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3871                                  EVENT_INDEX(5)));
3872         amdgpu_ring_write(ring, DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0));
3873
3874         /*
3875          * the address should be Qword aligned if 64bit write, Dword
3876          * aligned if only send 32bit data low (discard data high)
3877          */
3878         if (write64bit)
3879                 BUG_ON(addr & 0x7);
3880         else
3881                 BUG_ON(addr & 0x3);
3882         amdgpu_ring_write(ring, lower_32_bits(addr));
3883         amdgpu_ring_write(ring, upper_32_bits(addr));
3884         amdgpu_ring_write(ring, lower_32_bits(seq));
3885         amdgpu_ring_write(ring, upper_32_bits(seq));
3886         amdgpu_ring_write(ring, 0);
3887 }
3888
3889 static void gfx_v9_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
3890 {
3891         int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
3892         uint32_t seq = ring->fence_drv.sync_seq;
3893         uint64_t addr = ring->fence_drv.gpu_addr;
3894
3895         gfx_v9_0_wait_reg_mem(ring, usepfp, 1, 0,
3896                               lower_32_bits(addr), upper_32_bits(addr),
3897                               seq, 0xffffffff, 4);
3898 }
3899
3900 static void gfx_v9_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
3901                                         unsigned vmid, uint64_t pd_addr)
3902 {
3903         amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
3904
3905         /* compute doesn't have PFP */
3906         if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) {
3907                 /* sync PFP to ME, otherwise we might get invalid PFP reads */
3908                 amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
3909                 amdgpu_ring_write(ring, 0x0);
3910         }
3911 }
3912
3913 static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring)
3914 {
3915         return ring->adev->wb.wb[ring->rptr_offs]; /* gfx9 hardware is 32bit rptr */
3916 }
3917
3918 static u64 gfx_v9_0_ring_get_wptr_compute(struct amdgpu_ring *ring)
3919 {
3920         u64 wptr;
3921
3922         /* XXX check if swapping is necessary on BE */
3923         if (ring->use_doorbell)
3924                 wptr = atomic64_read((atomic64_t *)&ring->adev->wb.wb[ring->wptr_offs]);
3925         else
3926                 BUG();
3927         return wptr;
3928 }
3929
3930 static void gfx_v9_0_ring_set_pipe_percent(struct amdgpu_ring *ring,
3931                                            bool acquire)
3932 {
3933         struct amdgpu_device *adev = ring->adev;
3934         int pipe_num, tmp, reg;
3935         int pipe_percent = acquire ? SPI_WCL_PIPE_PERCENT_GFX__VALUE_MASK : 0x1;
3936
3937         pipe_num = ring->me * adev->gfx.mec.num_pipe_per_mec + ring->pipe;
3938
3939         /* first me only has 2 entries, GFX and HP3D */
3940         if (ring->me > 0)
3941                 pipe_num -= 2;
3942
3943         reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_GFX) + pipe_num;
3944         tmp = RREG32(reg);
3945         tmp = REG_SET_FIELD(tmp, SPI_WCL_PIPE_PERCENT_GFX, VALUE, pipe_percent);
3946         WREG32(reg, tmp);
3947 }
3948
3949 static void gfx_v9_0_pipe_reserve_resources(struct amdgpu_device *adev,
3950                                             struct amdgpu_ring *ring,
3951                                             bool acquire)
3952 {
3953         int i, pipe;
3954         bool reserve;
3955         struct amdgpu_ring *iring;
3956
3957         mutex_lock(&adev->gfx.pipe_reserve_mutex);
3958         pipe = amdgpu_gfx_queue_to_bit(adev, ring->me, ring->pipe, 0);
3959         if (acquire)
3960                 set_bit(pipe, adev->gfx.pipe_reserve_bitmap);
3961         else
3962                 clear_bit(pipe, adev->gfx.pipe_reserve_bitmap);
3963
3964         if (!bitmap_weight(adev->gfx.pipe_reserve_bitmap, AMDGPU_MAX_COMPUTE_QUEUES)) {
3965                 /* Clear all reservations - everyone reacquires all resources */
3966                 for (i = 0; i < adev->gfx.num_gfx_rings; ++i)
3967                         gfx_v9_0_ring_set_pipe_percent(&adev->gfx.gfx_ring[i],
3968                                                        true);
3969
3970                 for (i = 0; i < adev->gfx.num_compute_rings; ++i)
3971                         gfx_v9_0_ring_set_pipe_percent(&adev->gfx.compute_ring[i],
3972                                                        true);
3973         } else {
3974                 /* Lower all pipes without a current reservation */
3975                 for (i = 0; i < adev->gfx.num_gfx_rings; ++i) {
3976                         iring = &adev->gfx.gfx_ring[i];
3977                         pipe = amdgpu_gfx_queue_to_bit(adev,
3978                                                        iring->me,
3979                                                        iring->pipe,
3980                                                        0);
3981                         reserve = test_bit(pipe, adev->gfx.pipe_reserve_bitmap);
3982                         gfx_v9_0_ring_set_pipe_percent(iring, reserve);
3983                 }
3984
3985                 for (i = 0; i < adev->gfx.num_compute_rings; ++i) {
3986                         iring = &adev->gfx.compute_ring[i];
3987                         pipe = amdgpu_gfx_queue_to_bit(adev,
3988                                                        iring->me,
3989                                                        iring->pipe,
3990                                                        0);
3991                         reserve = test_bit(pipe, adev->gfx.pipe_reserve_bitmap);
3992                         gfx_v9_0_ring_set_pipe_percent(iring, reserve);
3993                 }
3994         }
3995
3996         mutex_unlock(&adev->gfx.pipe_reserve_mutex);
3997 }
3998
3999 static void gfx_v9_0_hqd_set_priority(struct amdgpu_device *adev,
4000                                       struct amdgpu_ring *ring,
4001                                       bool acquire)
4002 {
4003         uint32_t pipe_priority = acquire ? 0x2 : 0x0;
4004         uint32_t queue_priority = acquire ? 0xf : 0x0;
4005
4006         mutex_lock(&adev->srbm_mutex);
4007         soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
4008
4009         WREG32_SOC15(GC, 0, mmCP_HQD_PIPE_PRIORITY, pipe_priority);
4010         WREG32_SOC15(GC, 0, mmCP_HQD_QUEUE_PRIORITY, queue_priority);
4011
4012         soc15_grbm_select(adev, 0, 0, 0, 0);
4013         mutex_unlock(&adev->srbm_mutex);
4014 }
4015
4016 static void gfx_v9_0_ring_set_priority_compute(struct amdgpu_ring *ring,
4017                                                enum drm_sched_priority priority)
4018 {
4019         struct amdgpu_device *adev = ring->adev;
4020         bool acquire = priority == DRM_SCHED_PRIORITY_HIGH_HW;
4021
4022         if (ring->funcs->type != AMDGPU_RING_TYPE_COMPUTE)
4023                 return;
4024
4025         gfx_v9_0_hqd_set_priority(adev, ring, acquire);
4026         gfx_v9_0_pipe_reserve_resources(adev, ring, acquire);
4027 }
4028
4029 static void gfx_v9_0_ring_set_wptr_compute(struct amdgpu_ring *ring)
4030 {
4031         struct amdgpu_device *adev = ring->adev;
4032
4033         /* XXX check if swapping is necessary on BE */
4034         if (ring->use_doorbell) {
4035                 atomic64_set((atomic64_t*)&adev->wb.wb[ring->wptr_offs], ring->wptr);
4036                 WDOORBELL64(ring->doorbell_index, ring->wptr);
4037         } else{
4038                 BUG(); /* only DOORBELL method supported on gfx9 now */
4039         }
4040 }
4041
4042 static void gfx_v9_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr,
4043                                          u64 seq, unsigned int flags)
4044 {
4045         struct amdgpu_device *adev = ring->adev;
4046
4047         /* we only allocate 32bit for each seq wb address */
4048         BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
4049
4050         /* write fence seq to the "addr" */
4051         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4052         amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4053                                  WRITE_DATA_DST_SEL(5) | WR_CONFIRM));
4054         amdgpu_ring_write(ring, lower_32_bits(addr));
4055         amdgpu_ring_write(ring, upper_32_bits(addr));
4056         amdgpu_ring_write(ring, lower_32_bits(seq));
4057
4058         if (flags & AMDGPU_FENCE_FLAG_INT) {
4059                 /* set register to trigger INT */
4060                 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4061                 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4062                                          WRITE_DATA_DST_SEL(0) | WR_CONFIRM));
4063                 amdgpu_ring_write(ring, SOC15_REG_OFFSET(GC, 0, mmCPC_INT_STATUS));
4064                 amdgpu_ring_write(ring, 0);
4065                 amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */
4066         }
4067 }
4068
4069 static void gfx_v9_ring_emit_sb(struct amdgpu_ring *ring)
4070 {
4071         amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
4072         amdgpu_ring_write(ring, 0);
4073 }
4074
4075 static void gfx_v9_0_ring_emit_ce_meta(struct amdgpu_ring *ring)
4076 {
4077         struct v9_ce_ib_state ce_payload = {0};
4078         uint64_t csa_addr;
4079         int cnt;
4080
4081         cnt = (sizeof(ce_payload) >> 2) + 4 - 2;
4082         csa_addr = amdgpu_csa_vaddr(ring->adev);
4083
4084         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt));
4085         amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(2) |
4086                                  WRITE_DATA_DST_SEL(8) |
4087                                  WR_CONFIRM) |
4088                                  WRITE_DATA_CACHE_POLICY(0));
4089         amdgpu_ring_write(ring, lower_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, ce_payload)));
4090         amdgpu_ring_write(ring, upper_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, ce_payload)));
4091         amdgpu_ring_write_multiple(ring, (void *)&ce_payload, sizeof(ce_payload) >> 2);
4092 }
4093
4094 static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring)
4095 {
4096         struct v9_de_ib_state de_payload = {0};
4097         uint64_t csa_addr, gds_addr;
4098         int cnt;
4099
4100         csa_addr = amdgpu_csa_vaddr(ring->adev);
4101         gds_addr = csa_addr + 4096;
4102         de_payload.gds_backup_addrlo = lower_32_bits(gds_addr);
4103         de_payload.gds_backup_addrhi = upper_32_bits(gds_addr);
4104
4105         cnt = (sizeof(de_payload) >> 2) + 4 - 2;
4106         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt));
4107         amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
4108                                  WRITE_DATA_DST_SEL(8) |
4109                                  WR_CONFIRM) |
4110                                  WRITE_DATA_CACHE_POLICY(0));
4111         amdgpu_ring_write(ring, lower_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, de_payload)));
4112         amdgpu_ring_write(ring, upper_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, de_payload)));
4113         amdgpu_ring_write_multiple(ring, (void *)&de_payload, sizeof(de_payload) >> 2);
4114 }
4115
4116 static void gfx_v9_0_ring_emit_tmz(struct amdgpu_ring *ring, bool start)
4117 {
4118         amdgpu_ring_write(ring, PACKET3(PACKET3_FRAME_CONTROL, 0));
4119         amdgpu_ring_write(ring, FRAME_CMD(start ? 0 : 1)); /* frame_end */
4120 }
4121
4122 static void gfx_v9_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags)
4123 {
4124         uint32_t dw2 = 0;
4125
4126         if (amdgpu_sriov_vf(ring->adev))
4127                 gfx_v9_0_ring_emit_ce_meta(ring);
4128
4129         gfx_v9_0_ring_emit_tmz(ring, true);
4130
4131         dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */
4132         if (flags & AMDGPU_HAVE_CTX_SWITCH) {
4133                 /* set load_global_config & load_global_uconfig */
4134                 dw2 |= 0x8001;
4135                 /* set load_cs_sh_regs */
4136                 dw2 |= 0x01000000;
4137                 /* set load_per_context_state & load_gfx_sh_regs for GFX */
4138                 dw2 |= 0x10002;
4139
4140                 /* set load_ce_ram if preamble presented */
4141                 if (AMDGPU_PREAMBLE_IB_PRESENT & flags)
4142                         dw2 |= 0x10000000;
4143         } else {
4144                 /* still load_ce_ram if this is the first time preamble presented
4145                  * although there is no context switch happens.
4146                  */
4147                 if (AMDGPU_PREAMBLE_IB_PRESENT_FIRST & flags)
4148                         dw2 |= 0x10000000;
4149         }
4150
4151         amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
4152         amdgpu_ring_write(ring, dw2);
4153         amdgpu_ring_write(ring, 0);
4154 }
4155
4156 static unsigned gfx_v9_0_ring_emit_init_cond_exec(struct amdgpu_ring *ring)
4157 {
4158         unsigned ret;
4159         amdgpu_ring_write(ring, PACKET3(PACKET3_COND_EXEC, 3));
4160         amdgpu_ring_write(ring, lower_32_bits(ring->cond_exe_gpu_addr));
4161         amdgpu_ring_write(ring, upper_32_bits(ring->cond_exe_gpu_addr));
4162         amdgpu_ring_write(ring, 0); /* discard following DWs if *cond_exec_gpu_addr==0 */
4163         ret = ring->wptr & ring->buf_mask;
4164         amdgpu_ring_write(ring, 0x55aa55aa); /* patch dummy value later */
4165         return ret;
4166 }
4167
4168 static void gfx_v9_0_ring_emit_patch_cond_exec(struct amdgpu_ring *ring, unsigned offset)
4169 {
4170         unsigned cur;
4171         BUG_ON(offset > ring->buf_mask);
4172         BUG_ON(ring->ring[offset] != 0x55aa55aa);
4173
4174         cur = (ring->wptr & ring->buf_mask) - 1;
4175         if (likely(cur > offset))
4176                 ring->ring[offset] = cur - offset;
4177         else
4178                 ring->ring[offset] = (ring->ring_size>>2) - offset + cur;
4179 }
4180
4181 static void gfx_v9_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg)
4182 {
4183         struct amdgpu_device *adev = ring->adev;
4184
4185         amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));
4186         amdgpu_ring_write(ring, 0 |     /* src: register*/
4187                                 (5 << 8) |      /* dst: memory */
4188                                 (1 << 20));     /* write confirm */
4189         amdgpu_ring_write(ring, reg);
4190         amdgpu_ring_write(ring, 0);
4191         amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +
4192                                 adev->virt.reg_val_offs * 4));
4193         amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +
4194                                 adev->virt.reg_val_offs * 4));
4195 }
4196
4197 static void gfx_v9_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg,
4198                                     uint32_t val)
4199 {
4200         uint32_t cmd = 0;
4201
4202         switch (ring->funcs->type) {
4203         case AMDGPU_RING_TYPE_GFX:
4204                 cmd = WRITE_DATA_ENGINE_SEL(1) | WR_CONFIRM;
4205                 break;
4206         case AMDGPU_RING_TYPE_KIQ:
4207                 cmd = (1 << 16); /* no inc addr */
4208                 break;
4209         default:
4210                 cmd = WR_CONFIRM;
4211                 break;
4212         }
4213         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4214         amdgpu_ring_write(ring, cmd);
4215         amdgpu_ring_write(ring, reg);
4216         amdgpu_ring_write(ring, 0);
4217         amdgpu_ring_write(ring, val);
4218 }
4219
4220 static void gfx_v9_0_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
4221                                         uint32_t val, uint32_t mask)
4222 {
4223         gfx_v9_0_wait_reg_mem(ring, 0, 0, 0, reg, 0, val, mask, 0x20);
4224 }
4225
4226 static void gfx_v9_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring,
4227                                                   uint32_t reg0, uint32_t reg1,
4228                                                   uint32_t ref, uint32_t mask)
4229 {
4230         int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
4231
4232         if (amdgpu_sriov_vf(ring->adev))
4233                 gfx_v9_0_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1,
4234                                       ref, mask, 0x20);
4235         else
4236                 amdgpu_ring_emit_reg_write_reg_wait_helper(ring, reg0, reg1,
4237                                                            ref, mask);
4238 }
4239
4240 static void gfx_v9_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev,
4241                                                  enum amdgpu_interrupt_state state)
4242 {
4243         switch (state) {
4244         case AMDGPU_IRQ_STATE_DISABLE:
4245         case AMDGPU_IRQ_STATE_ENABLE:
4246                 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
4247                                TIME_STAMP_INT_ENABLE,
4248                                state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
4249                 break;
4250         default:
4251                 break;
4252         }
4253 }
4254
4255 static void gfx_v9_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev,
4256                                                      int me, int pipe,
4257                                                      enum amdgpu_interrupt_state state)
4258 {
4259         u32 mec_int_cntl, mec_int_cntl_reg;
4260
4261         /*
4262          * amdgpu controls only the first MEC. That's why this function only
4263          * handles the setting of interrupts for this specific MEC. All other
4264          * pipes' interrupts are set by amdkfd.
4265          */
4266
4267         if (me == 1) {
4268                 switch (pipe) {
4269                 case 0:
4270                         mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE0_INT_CNTL);
4271                         break;
4272                 case 1:
4273                         mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE1_INT_CNTL);
4274                         break;
4275                 case 2:
4276                         mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE2_INT_CNTL);
4277                         break;
4278                 case 3:
4279                         mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE3_INT_CNTL);
4280                         break;
4281                 default:
4282                         DRM_DEBUG("invalid pipe %d\n", pipe);
4283                         return;
4284                 }
4285         } else {
4286                 DRM_DEBUG("invalid me %d\n", me);
4287                 return;
4288         }
4289
4290         switch (state) {
4291         case AMDGPU_IRQ_STATE_DISABLE:
4292                 mec_int_cntl = RREG32(mec_int_cntl_reg);
4293                 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
4294                                              TIME_STAMP_INT_ENABLE, 0);
4295                 WREG32(mec_int_cntl_reg, mec_int_cntl);
4296                 break;
4297         case AMDGPU_IRQ_STATE_ENABLE:
4298                 mec_int_cntl = RREG32(mec_int_cntl_reg);
4299                 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
4300                                              TIME_STAMP_INT_ENABLE, 1);
4301                 WREG32(mec_int_cntl_reg, mec_int_cntl);
4302                 break;
4303         default:
4304                 break;
4305         }
4306 }
4307
4308 static int gfx_v9_0_set_priv_reg_fault_state(struct amdgpu_device *adev,
4309                                              struct amdgpu_irq_src *source,
4310                                              unsigned type,
4311                                              enum amdgpu_interrupt_state state)
4312 {
4313         switch (state) {
4314         case AMDGPU_IRQ_STATE_DISABLE:
4315         case AMDGPU_IRQ_STATE_ENABLE:
4316                 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
4317                                PRIV_REG_INT_ENABLE,
4318                                state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
4319                 break;
4320         default:
4321                 break;
4322         }
4323
4324         return 0;
4325 }
4326
4327 static int gfx_v9_0_set_priv_inst_fault_state(struct amdgpu_device *adev,
4328                                               struct amdgpu_irq_src *source,
4329                                               unsigned type,
4330                                               enum amdgpu_interrupt_state state)
4331 {
4332         switch (state) {
4333         case AMDGPU_IRQ_STATE_DISABLE:
4334         case AMDGPU_IRQ_STATE_ENABLE:
4335                 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
4336                                PRIV_INSTR_INT_ENABLE,
4337                                state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
4338         default:
4339                 break;
4340         }
4341
4342         return 0;
4343 }
4344
4345 static int gfx_v9_0_set_eop_interrupt_state(struct amdgpu_device *adev,
4346                                             struct amdgpu_irq_src *src,
4347                                             unsigned type,
4348                                             enum amdgpu_interrupt_state state)
4349 {
4350         switch (type) {
4351         case AMDGPU_CP_IRQ_GFX_EOP:
4352                 gfx_v9_0_set_gfx_eop_interrupt_state(adev, state);
4353                 break;
4354         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP:
4355                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 0, state);
4356                 break;
4357         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP:
4358                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 1, state);
4359                 break;
4360         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP:
4361                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 2, state);
4362                 break;
4363         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP:
4364                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 3, state);
4365                 break;
4366         case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE0_EOP:
4367                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 0, state);
4368                 break;
4369         case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE1_EOP:
4370                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 1, state);
4371                 break;
4372         case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE2_EOP:
4373                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 2, state);
4374                 break;
4375         case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE3_EOP:
4376                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 3, state);
4377                 break;
4378         default:
4379                 break;
4380         }
4381         return 0;
4382 }
4383
4384 static int gfx_v9_0_eop_irq(struct amdgpu_device *adev,
4385                             struct amdgpu_irq_src *source,
4386                             struct amdgpu_iv_entry *entry)
4387 {
4388         int i;
4389         u8 me_id, pipe_id, queue_id;
4390         struct amdgpu_ring *ring;
4391
4392         DRM_DEBUG("IH: CP EOP\n");
4393         me_id = (entry->ring_id & 0x0c) >> 2;
4394         pipe_id = (entry->ring_id & 0x03) >> 0;
4395         queue_id = (entry->ring_id & 0x70) >> 4;
4396
4397         switch (me_id) {
4398         case 0:
4399                 amdgpu_fence_process(&adev->gfx.gfx_ring[0]);
4400                 break;
4401         case 1:
4402         case 2:
4403                 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
4404                         ring = &adev->gfx.compute_ring[i];
4405                         /* Per-queue interrupt is supported for MEC starting from VI.
4406                           * The interrupt can only be enabled/disabled per pipe instead of per queue.
4407                           */
4408                         if ((ring->me == me_id) && (ring->pipe == pipe_id) && (ring->queue == queue_id))
4409                                 amdgpu_fence_process(ring);
4410                 }
4411                 break;
4412         }
4413         return 0;
4414 }
4415
4416 static int gfx_v9_0_priv_reg_irq(struct amdgpu_device *adev,
4417                                  struct amdgpu_irq_src *source,
4418                                  struct amdgpu_iv_entry *entry)
4419 {
4420         DRM_ERROR("Illegal register access in command stream\n");
4421         schedule_work(&adev->reset_work);
4422         return 0;
4423 }
4424
4425 static int gfx_v9_0_priv_inst_irq(struct amdgpu_device *adev,
4426                                   struct amdgpu_irq_src *source,
4427                                   struct amdgpu_iv_entry *entry)
4428 {
4429         DRM_ERROR("Illegal instruction in command stream\n");
4430         schedule_work(&adev->reset_work);
4431         return 0;
4432 }
4433
4434 static int gfx_v9_0_kiq_set_interrupt_state(struct amdgpu_device *adev,
4435                                             struct amdgpu_irq_src *src,
4436                                             unsigned int type,
4437                                             enum amdgpu_interrupt_state state)
4438 {
4439         uint32_t tmp, target;
4440         struct amdgpu_ring *ring = &(adev->gfx.kiq.ring);
4441
4442         if (ring->me == 1)
4443                 target = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE0_INT_CNTL);
4444         else
4445                 target = SOC15_REG_OFFSET(GC, 0, mmCP_ME2_PIPE0_INT_CNTL);
4446         target += ring->pipe;
4447
4448         switch (type) {
4449         case AMDGPU_CP_KIQ_IRQ_DRIVER0:
4450                 if (state == AMDGPU_IRQ_STATE_DISABLE) {
4451                         tmp = RREG32_SOC15(GC, 0, mmCPC_INT_CNTL);
4452                         tmp = REG_SET_FIELD(tmp, CPC_INT_CNTL,
4453                                                  GENERIC2_INT_ENABLE, 0);
4454                         WREG32_SOC15(GC, 0, mmCPC_INT_CNTL, tmp);
4455
4456                         tmp = RREG32(target);
4457                         tmp = REG_SET_FIELD(tmp, CP_ME2_PIPE0_INT_CNTL,
4458                                                  GENERIC2_INT_ENABLE, 0);
4459                         WREG32(target, tmp);
4460                 } else {
4461                         tmp = RREG32_SOC15(GC, 0, mmCPC_INT_CNTL);
4462                         tmp = REG_SET_FIELD(tmp, CPC_INT_CNTL,
4463                                                  GENERIC2_INT_ENABLE, 1);
4464                         WREG32_SOC15(GC, 0, mmCPC_INT_CNTL, tmp);
4465
4466                         tmp = RREG32(target);
4467                         tmp = REG_SET_FIELD(tmp, CP_ME2_PIPE0_INT_CNTL,
4468                                                  GENERIC2_INT_ENABLE, 1);
4469                         WREG32(target, tmp);
4470                 }
4471                 break;
4472         default:
4473                 BUG(); /* kiq only support GENERIC2_INT now */
4474                 break;
4475         }
4476         return 0;
4477 }
4478
4479 static int gfx_v9_0_kiq_irq(struct amdgpu_device *adev,
4480                             struct amdgpu_irq_src *source,
4481                             struct amdgpu_iv_entry *entry)
4482 {
4483         u8 me_id, pipe_id, queue_id;
4484         struct amdgpu_ring *ring = &(adev->gfx.kiq.ring);
4485
4486         me_id = (entry->ring_id & 0x0c) >> 2;
4487         pipe_id = (entry->ring_id & 0x03) >> 0;
4488         queue_id = (entry->ring_id & 0x70) >> 4;
4489         DRM_DEBUG("IH: CPC GENERIC2_INT, me:%d, pipe:%d, queue:%d\n",
4490                    me_id, pipe_id, queue_id);
4491
4492         amdgpu_fence_process(ring);
4493         return 0;
4494 }
4495
4496 static const struct amd_ip_funcs gfx_v9_0_ip_funcs = {
4497         .name = "gfx_v9_0",
4498         .early_init = gfx_v9_0_early_init,
4499         .late_init = gfx_v9_0_late_init,
4500         .sw_init = gfx_v9_0_sw_init,
4501         .sw_fini = gfx_v9_0_sw_fini,
4502         .hw_init = gfx_v9_0_hw_init,
4503         .hw_fini = gfx_v9_0_hw_fini,
4504         .suspend = gfx_v9_0_suspend,
4505         .resume = gfx_v9_0_resume,
4506         .is_idle = gfx_v9_0_is_idle,
4507         .wait_for_idle = gfx_v9_0_wait_for_idle,
4508         .soft_reset = gfx_v9_0_soft_reset,
4509         .set_clockgating_state = gfx_v9_0_set_clockgating_state,
4510         .set_powergating_state = gfx_v9_0_set_powergating_state,
4511         .get_clockgating_state = gfx_v9_0_get_clockgating_state,
4512 };
4513
4514 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_gfx = {
4515         .type = AMDGPU_RING_TYPE_GFX,
4516         .align_mask = 0xff,
4517         .nop = PACKET3(PACKET3_NOP, 0x3FFF),
4518         .support_64bit_ptrs = true,
4519         .vmhub = AMDGPU_GFXHUB,
4520         .get_rptr = gfx_v9_0_ring_get_rptr_gfx,
4521         .get_wptr = gfx_v9_0_ring_get_wptr_gfx,
4522         .set_wptr = gfx_v9_0_ring_set_wptr_gfx,
4523         .emit_frame_size = /* totally 242 maximum if 16 IBs */
4524                 5 +  /* COND_EXEC */
4525                 7 +  /* PIPELINE_SYNC */
4526                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
4527                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
4528                 2 + /* VM_FLUSH */
4529                 8 +  /* FENCE for VM_FLUSH */
4530                 20 + /* GDS switch */
4531                 4 + /* double SWITCH_BUFFER,
4532                        the first COND_EXEC jump to the place just
4533                            prior to this double SWITCH_BUFFER  */
4534                 5 + /* COND_EXEC */
4535                 7 +      /*     HDP_flush */
4536                 4 +      /*     VGT_flush */
4537                 14 + /* CE_META */
4538                 31 + /* DE_META */
4539                 3 + /* CNTX_CTRL */
4540                 5 + /* HDP_INVL */
4541                 8 + 8 + /* FENCE x2 */
4542                 2, /* SWITCH_BUFFER */
4543         .emit_ib_size = 4, /* gfx_v9_0_ring_emit_ib_gfx */
4544         .emit_ib = gfx_v9_0_ring_emit_ib_gfx,
4545         .emit_fence = gfx_v9_0_ring_emit_fence,
4546         .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync,
4547         .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush,
4548         .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch,
4549         .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush,
4550         .test_ring = gfx_v9_0_ring_test_ring,
4551         .test_ib = gfx_v9_0_ring_test_ib,
4552         .insert_nop = amdgpu_ring_insert_nop,
4553         .pad_ib = amdgpu_ring_generic_pad_ib,
4554         .emit_switch_buffer = gfx_v9_ring_emit_sb,
4555         .emit_cntxcntl = gfx_v9_ring_emit_cntxcntl,
4556         .init_cond_exec = gfx_v9_0_ring_emit_init_cond_exec,
4557         .patch_cond_exec = gfx_v9_0_ring_emit_patch_cond_exec,
4558         .emit_tmz = gfx_v9_0_ring_emit_tmz,
4559         .emit_wreg = gfx_v9_0_ring_emit_wreg,
4560         .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,
4561         .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,
4562 };
4563
4564 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_compute = {
4565         .type = AMDGPU_RING_TYPE_COMPUTE,
4566         .align_mask = 0xff,
4567         .nop = PACKET3(PACKET3_NOP, 0x3FFF),
4568         .support_64bit_ptrs = true,
4569         .vmhub = AMDGPU_GFXHUB,
4570         .get_rptr = gfx_v9_0_ring_get_rptr_compute,
4571         .get_wptr = gfx_v9_0_ring_get_wptr_compute,
4572         .set_wptr = gfx_v9_0_ring_set_wptr_compute,
4573         .emit_frame_size =
4574                 20 + /* gfx_v9_0_ring_emit_gds_switch */
4575                 7 + /* gfx_v9_0_ring_emit_hdp_flush */
4576                 5 + /* hdp invalidate */
4577                 7 + /* gfx_v9_0_ring_emit_pipeline_sync */
4578                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
4579                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
4580                 2 + /* gfx_v9_0_ring_emit_vm_flush */
4581                 8 + 8 + 8, /* gfx_v9_0_ring_emit_fence x3 for user fence, vm fence */
4582         .emit_ib_size = 4, /* gfx_v9_0_ring_emit_ib_compute */
4583         .emit_ib = gfx_v9_0_ring_emit_ib_compute,
4584         .emit_fence = gfx_v9_0_ring_emit_fence,
4585         .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync,
4586         .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush,
4587         .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch,
4588         .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush,
4589         .test_ring = gfx_v9_0_ring_test_ring,
4590         .test_ib = gfx_v9_0_ring_test_ib,
4591         .insert_nop = amdgpu_ring_insert_nop,
4592         .pad_ib = amdgpu_ring_generic_pad_ib,
4593         .set_priority = gfx_v9_0_ring_set_priority_compute,
4594         .emit_wreg = gfx_v9_0_ring_emit_wreg,
4595         .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,
4596         .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,
4597 };
4598
4599 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_kiq = {
4600         .type = AMDGPU_RING_TYPE_KIQ,
4601         .align_mask = 0xff,
4602         .nop = PACKET3(PACKET3_NOP, 0x3FFF),
4603         .support_64bit_ptrs = true,
4604         .vmhub = AMDGPU_GFXHUB,
4605         .get_rptr = gfx_v9_0_ring_get_rptr_compute,
4606         .get_wptr = gfx_v9_0_ring_get_wptr_compute,
4607         .set_wptr = gfx_v9_0_ring_set_wptr_compute,
4608         .emit_frame_size =
4609                 20 + /* gfx_v9_0_ring_emit_gds_switch */
4610                 7 + /* gfx_v9_0_ring_emit_hdp_flush */
4611                 5 + /* hdp invalidate */
4612                 7 + /* gfx_v9_0_ring_emit_pipeline_sync */
4613                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
4614                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
4615                 2 + /* gfx_v9_0_ring_emit_vm_flush */
4616                 8 + 8 + 8, /* gfx_v9_0_ring_emit_fence_kiq x3 for user fence, vm fence */
4617         .emit_ib_size = 4, /* gfx_v9_0_ring_emit_ib_compute */
4618         .emit_ib = gfx_v9_0_ring_emit_ib_compute,
4619         .emit_fence = gfx_v9_0_ring_emit_fence_kiq,
4620         .test_ring = gfx_v9_0_ring_test_ring,
4621         .test_ib = gfx_v9_0_ring_test_ib,
4622         .insert_nop = amdgpu_ring_insert_nop,
4623         .pad_ib = amdgpu_ring_generic_pad_ib,
4624         .emit_rreg = gfx_v9_0_ring_emit_rreg,
4625         .emit_wreg = gfx_v9_0_ring_emit_wreg,
4626         .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,
4627         .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,
4628 };
4629
4630 static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev)
4631 {
4632         int i;
4633
4634         adev->gfx.kiq.ring.funcs = &gfx_v9_0_ring_funcs_kiq;
4635
4636         for (i = 0; i < adev->gfx.num_gfx_rings; i++)
4637                 adev->gfx.gfx_ring[i].funcs = &gfx_v9_0_ring_funcs_gfx;
4638
4639         for (i = 0; i < adev->gfx.num_compute_rings; i++)
4640                 adev->gfx.compute_ring[i].funcs = &gfx_v9_0_ring_funcs_compute;
4641 }
4642
4643 static const struct amdgpu_irq_src_funcs gfx_v9_0_kiq_irq_funcs = {
4644         .set = gfx_v9_0_kiq_set_interrupt_state,
4645         .process = gfx_v9_0_kiq_irq,
4646 };
4647
4648 static const struct amdgpu_irq_src_funcs gfx_v9_0_eop_irq_funcs = {
4649         .set = gfx_v9_0_set_eop_interrupt_state,
4650         .process = gfx_v9_0_eop_irq,
4651 };
4652
4653 static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_reg_irq_funcs = {
4654         .set = gfx_v9_0_set_priv_reg_fault_state,
4655         .process = gfx_v9_0_priv_reg_irq,
4656 };
4657
4658 static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_inst_irq_funcs = {
4659         .set = gfx_v9_0_set_priv_inst_fault_state,
4660         .process = gfx_v9_0_priv_inst_irq,
4661 };
4662
4663 static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev)
4664 {
4665         adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST;
4666         adev->gfx.eop_irq.funcs = &gfx_v9_0_eop_irq_funcs;
4667
4668         adev->gfx.priv_reg_irq.num_types = 1;
4669         adev->gfx.priv_reg_irq.funcs = &gfx_v9_0_priv_reg_irq_funcs;
4670
4671         adev->gfx.priv_inst_irq.num_types = 1;
4672         adev->gfx.priv_inst_irq.funcs = &gfx_v9_0_priv_inst_irq_funcs;
4673
4674         adev->gfx.kiq.irq.num_types = AMDGPU_CP_KIQ_IRQ_LAST;
4675         adev->gfx.kiq.irq.funcs = &gfx_v9_0_kiq_irq_funcs;
4676 }
4677
4678 static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev)
4679 {
4680         switch (adev->asic_type) {
4681         case CHIP_VEGA10:
4682         case CHIP_VEGA12:
4683         case CHIP_RAVEN:
4684                 adev->gfx.rlc.funcs = &gfx_v9_0_rlc_funcs;
4685                 break;
4686         default:
4687                 break;
4688         }
4689 }
4690
4691 static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev)
4692 {
4693         /* init asci gds info */
4694         adev->gds.mem.total_size = RREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE);
4695         adev->gds.gws.total_size = 64;
4696         adev->gds.oa.total_size = 16;
4697
4698         if (adev->gds.mem.total_size == 64 * 1024) {
4699                 adev->gds.mem.gfx_partition_size = 4096;
4700                 adev->gds.mem.cs_partition_size = 4096;
4701
4702                 adev->gds.gws.gfx_partition_size = 4;
4703                 adev->gds.gws.cs_partition_size = 4;
4704
4705                 adev->gds.oa.gfx_partition_size = 4;
4706                 adev->gds.oa.cs_partition_size = 1;
4707         } else {
4708                 adev->gds.mem.gfx_partition_size = 1024;
4709                 adev->gds.mem.cs_partition_size = 1024;
4710
4711                 adev->gds.gws.gfx_partition_size = 16;
4712                 adev->gds.gws.cs_partition_size = 16;
4713
4714                 adev->gds.oa.gfx_partition_size = 4;
4715                 adev->gds.oa.cs_partition_size = 4;
4716         }
4717 }
4718
4719 static void gfx_v9_0_set_user_cu_inactive_bitmap(struct amdgpu_device *adev,
4720                                                  u32 bitmap)
4721 {
4722         u32 data;
4723
4724         if (!bitmap)
4725                 return;
4726
4727         data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT;
4728         data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK;
4729
4730         WREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG, data);
4731 }
4732
4733 static u32 gfx_v9_0_get_cu_active_bitmap(struct amdgpu_device *adev)
4734 {
4735         u32 data, mask;
4736
4737         data = RREG32_SOC15(GC, 0, mmCC_GC_SHADER_ARRAY_CONFIG);
4738         data |= RREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG);
4739
4740         data &= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK;
4741         data >>= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT;
4742
4743         mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh);
4744
4745         return (~data) & mask;
4746 }
4747
4748 static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev,
4749                                  struct amdgpu_cu_info *cu_info)
4750 {
4751         int i, j, k, counter, active_cu_number = 0;
4752         u32 mask, bitmap, ao_bitmap, ao_cu_mask = 0;
4753         unsigned disable_masks[4 * 2];
4754
4755         if (!adev || !cu_info)
4756                 return -EINVAL;
4757
4758         amdgpu_gfx_parse_disable_cu(disable_masks, 4, 2);
4759
4760         mutex_lock(&adev->grbm_idx_mutex);
4761         for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
4762                 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
4763                         mask = 1;
4764                         ao_bitmap = 0;
4765                         counter = 0;
4766                         gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff);
4767                         if (i < 4 && j < 2)
4768                                 gfx_v9_0_set_user_cu_inactive_bitmap(
4769                                         adev, disable_masks[i * 2 + j]);
4770                         bitmap = gfx_v9_0_get_cu_active_bitmap(adev);
4771                         cu_info->bitmap[i][j] = bitmap;
4772
4773                         for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) {
4774                                 if (bitmap & mask) {
4775                                         if (counter < adev->gfx.config.max_cu_per_sh)
4776                                                 ao_bitmap |= mask;
4777                                         counter ++;
4778                                 }
4779                                 mask <<= 1;
4780                         }
4781                         active_cu_number += counter;
4782                         if (i < 2 && j < 2)
4783                                 ao_cu_mask |= (ao_bitmap << (i * 16 + j * 8));
4784                         cu_info->ao_cu_bitmap[i][j] = ao_bitmap;
4785                 }
4786         }
4787         gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
4788         mutex_unlock(&adev->grbm_idx_mutex);
4789
4790         cu_info->number = active_cu_number;
4791         cu_info->ao_cu_mask = ao_cu_mask;
4792
4793         return 0;
4794 }
4795
4796 const struct amdgpu_ip_block_version gfx_v9_0_ip_block =
4797 {
4798         .type = AMD_IP_BLOCK_TYPE_GFX,
4799         .major = 9,
4800         .minor = 0,
4801         .rev = 0,
4802         .funcs = &gfx_v9_0_ip_funcs,
4803 };