drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c

   1 /*
   2  * Copyright 2014 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23
  24 #include <linux/delay.h>
  25 #include <linux/kernel.h>
  26 #include <linux/firmware.h>
  27 #include <linux/module.h>
  28 #include <linux/pci.h>
  29
  30 #include "amdgpu.h"
  31 #include "amdgpu_gfx.h"
  32 #include "amdgpu_ring.h"
  33 #include "vi.h"
  34 #include "vi_structs.h"
  35 #include "vid.h"
  36 #include "amdgpu_ucode.h"
  37 #include "amdgpu_atombios.h"
  38 #include "atombios_i2c.h"
  39 #include "clearstate_vi.h"
  40
  41 #include "gmc/gmc_8_2_d.h"
  42 #include "gmc/gmc_8_2_sh_mask.h"
  43
  44 #include "oss/oss_3_0_d.h"
  45 #include "oss/oss_3_0_sh_mask.h"
  46
  47 #include "bif/bif_5_0_d.h"
  48 #include "bif/bif_5_0_sh_mask.h"
  49 #include "gca/gfx_8_0_d.h"
  50 #include "gca/gfx_8_0_enum.h"
  51 #include "gca/gfx_8_0_sh_mask.h"
  52
  53 #include "dce/dce_10_0_d.h"
  54 #include "dce/dce_10_0_sh_mask.h"
  55
  56 #include "smu/smu_7_1_3_d.h"
  57
  58 #include "ivsrcid/ivsrcid_vislands30.h"
  59
  60 #define GFX8_NUM_GFX_RINGS     1
  61 #define GFX8_MEC_HPD_SIZE 4096
  62
  63 #define TOPAZ_GB_ADDR_CONFIG_GOLDEN 0x22010001
  64 #define CARRIZO_GB_ADDR_CONFIG_GOLDEN 0x22010001
  65 #define POLARIS11_GB_ADDR_CONFIG_GOLDEN 0x22011002
  66 #define TONGA_GB_ADDR_CONFIG_GOLDEN 0x22011003
  67
  68 #define ARRAY_MODE(x)                                   ((x) << GB_TILE_MODE0__ARRAY_MODE__SHIFT)
  69 #define PIPE_CONFIG(x)                                  ((x) << GB_TILE_MODE0__PIPE_CONFIG__SHIFT)
  70 #define TILE_SPLIT(x)                                   ((x) << GB_TILE_MODE0__TILE_SPLIT__SHIFT)
  71 #define MICRO_TILE_MODE_NEW(x)                          ((x) << GB_TILE_MODE0__MICRO_TILE_MODE_NEW__SHIFT)
  72 #define SAMPLE_SPLIT(x)                                 ((x) << GB_TILE_MODE0__SAMPLE_SPLIT__SHIFT)
  73 #define BANK_WIDTH(x)                                   ((x) << GB_MACROTILE_MODE0__BANK_WIDTH__SHIFT)
  74 #define BANK_HEIGHT(x)                                  ((x) << GB_MACROTILE_MODE0__BANK_HEIGHT__SHIFT)
  75 #define MACRO_TILE_ASPECT(x)                            ((x) << GB_MACROTILE_MODE0__MACRO_TILE_ASPECT__SHIFT)
  76 #define NUM_BANKS(x)                                    ((x) << GB_MACROTILE_MODE0__NUM_BANKS__SHIFT)
  77
  78 #define RLC_CGTT_MGCG_OVERRIDE__CPF_MASK            0x00000001L
  79 #define RLC_CGTT_MGCG_OVERRIDE__RLC_MASK            0x00000002L
  80 #define RLC_CGTT_MGCG_OVERRIDE__MGCG_MASK           0x00000004L
  81 #define RLC_CGTT_MGCG_OVERRIDE__CGCG_MASK           0x00000008L
  82 #define RLC_CGTT_MGCG_OVERRIDE__CGLS_MASK           0x00000010L
  83 #define RLC_CGTT_MGCG_OVERRIDE__GRBM_MASK           0x00000020L
  84
  85 /* BPM SERDES CMD */
  86 #define SET_BPM_SERDES_CMD    1
  87 #define CLE_BPM_SERDES_CMD    0
  88
  89 /* BPM Register Address*/
  90 enum {
  91         BPM_REG_CGLS_EN = 0,        /* Enable/Disable CGLS */
  92         BPM_REG_CGLS_ON,            /* ON/OFF CGLS: shall be controlled by RLC FW */
  93         BPM_REG_CGCG_OVERRIDE,      /* Set/Clear CGCG Override */
  94         BPM_REG_MGCG_OVERRIDE,      /* Set/Clear MGCG Override */
  95         BPM_REG_FGCG_OVERRIDE,      /* Set/Clear FGCG Override */
  96         BPM_REG_FGCG_MAX
  97 };
  98
  99 #define RLC_FormatDirectRegListLength        14
 100
 101 MODULE_FIRMWARE("amdgpu/carrizo_ce.bin");
 102 MODULE_FIRMWARE("amdgpu/carrizo_pfp.bin");
 103 MODULE_FIRMWARE("amdgpu/carrizo_me.bin");
 104 MODULE_FIRMWARE("amdgpu/carrizo_mec.bin");
 105 MODULE_FIRMWARE("amdgpu/carrizo_mec2.bin");
 106 MODULE_FIRMWARE("amdgpu/carrizo_rlc.bin");
 107
 108 MODULE_FIRMWARE("amdgpu/stoney_ce.bin");
 109 MODULE_FIRMWARE("amdgpu/stoney_pfp.bin");
 110 MODULE_FIRMWARE("amdgpu/stoney_me.bin");
 111 MODULE_FIRMWARE("amdgpu/stoney_mec.bin");
 112 MODULE_FIRMWARE("amdgpu/stoney_rlc.bin");
 113
 114 MODULE_FIRMWARE("amdgpu/tonga_ce.bin");
 115 MODULE_FIRMWARE("amdgpu/tonga_pfp.bin");
 116 MODULE_FIRMWARE("amdgpu/tonga_me.bin");
 117 MODULE_FIRMWARE("amdgpu/tonga_mec.bin");
 118 MODULE_FIRMWARE("amdgpu/tonga_mec2.bin");
 119 MODULE_FIRMWARE("amdgpu/tonga_rlc.bin");
 120
 121 MODULE_FIRMWARE("amdgpu/topaz_ce.bin");
 122 MODULE_FIRMWARE("amdgpu/topaz_pfp.bin");
 123 MODULE_FIRMWARE("amdgpu/topaz_me.bin");
 124 MODULE_FIRMWARE("amdgpu/topaz_mec.bin");
 125 MODULE_FIRMWARE("amdgpu/topaz_rlc.bin");
 126
 127 MODULE_FIRMWARE("amdgpu/fiji_ce.bin");
 128 MODULE_FIRMWARE("amdgpu/fiji_pfp.bin");
 129 MODULE_FIRMWARE("amdgpu/fiji_me.bin");
 130 MODULE_FIRMWARE("amdgpu/fiji_mec.bin");
 131 MODULE_FIRMWARE("amdgpu/fiji_mec2.bin");
 132 MODULE_FIRMWARE("amdgpu/fiji_rlc.bin");
 133
 134 MODULE_FIRMWARE("amdgpu/polaris10_ce.bin");
 135 MODULE_FIRMWARE("amdgpu/polaris10_ce_2.bin");
 136 MODULE_FIRMWARE("amdgpu/polaris10_pfp.bin");
 137 MODULE_FIRMWARE("amdgpu/polaris10_pfp_2.bin");
 138 MODULE_FIRMWARE("amdgpu/polaris10_me.bin");
 139 MODULE_FIRMWARE("amdgpu/polaris10_me_2.bin");
 140 MODULE_FIRMWARE("amdgpu/polaris10_mec.bin");
 141 MODULE_FIRMWARE("amdgpu/polaris10_mec_2.bin");
 142 MODULE_FIRMWARE("amdgpu/polaris10_mec2.bin");
 143 MODULE_FIRMWARE("amdgpu/polaris10_mec2_2.bin");
 144 MODULE_FIRMWARE("amdgpu/polaris10_rlc.bin");
 145
 146 MODULE_FIRMWARE("amdgpu/polaris11_ce.bin");
 147 MODULE_FIRMWARE("amdgpu/polaris11_ce_2.bin");
 148 MODULE_FIRMWARE("amdgpu/polaris11_pfp.bin");
 149 MODULE_FIRMWARE("amdgpu/polaris11_pfp_2.bin");
 150 MODULE_FIRMWARE("amdgpu/polaris11_me.bin");
 151 MODULE_FIRMWARE("amdgpu/polaris11_me_2.bin");
 152 MODULE_FIRMWARE("amdgpu/polaris11_mec.bin");
 153 MODULE_FIRMWARE("amdgpu/polaris11_mec_2.bin");
 154 MODULE_FIRMWARE("amdgpu/polaris11_mec2.bin");
 155 MODULE_FIRMWARE("amdgpu/polaris11_mec2_2.bin");
 156 MODULE_FIRMWARE("amdgpu/polaris11_rlc.bin");
 157
 158 MODULE_FIRMWARE("amdgpu/polaris12_ce.bin");
 159 MODULE_FIRMWARE("amdgpu/polaris12_ce_2.bin");
 160 MODULE_FIRMWARE("amdgpu/polaris12_pfp.bin");
 161 MODULE_FIRMWARE("amdgpu/polaris12_pfp_2.bin");
 162 MODULE_FIRMWARE("amdgpu/polaris12_me.bin");
 163 MODULE_FIRMWARE("amdgpu/polaris12_me_2.bin");
 164 MODULE_FIRMWARE("amdgpu/polaris12_mec.bin");
 165 MODULE_FIRMWARE("amdgpu/polaris12_mec_2.bin");
 166 MODULE_FIRMWARE("amdgpu/polaris12_mec2.bin");
 167 MODULE_FIRMWARE("amdgpu/polaris12_mec2_2.bin");
 168 MODULE_FIRMWARE("amdgpu/polaris12_rlc.bin");
 169
 170 MODULE_FIRMWARE("amdgpu/vegam_ce.bin");
 171 MODULE_FIRMWARE("amdgpu/vegam_pfp.bin");
 172 MODULE_FIRMWARE("amdgpu/vegam_me.bin");
 173 MODULE_FIRMWARE("amdgpu/vegam_mec.bin");
 174 MODULE_FIRMWARE("amdgpu/vegam_mec2.bin");
 175 MODULE_FIRMWARE("amdgpu/vegam_rlc.bin");
 176
 177 static const struct amdgpu_gds_reg_offset amdgpu_gds_reg_offset[] =
 178 {
 179         {mmGDS_VMID0_BASE, mmGDS_VMID0_SIZE, mmGDS_GWS_VMID0, mmGDS_OA_VMID0},
 180         {mmGDS_VMID1_BASE, mmGDS_VMID1_SIZE, mmGDS_GWS_VMID1, mmGDS_OA_VMID1},
 181         {mmGDS_VMID2_BASE, mmGDS_VMID2_SIZE, mmGDS_GWS_VMID2, mmGDS_OA_VMID2},
 182         {mmGDS_VMID3_BASE, mmGDS_VMID3_SIZE, mmGDS_GWS_VMID3, mmGDS_OA_VMID3},
 183         {mmGDS_VMID4_BASE, mmGDS_VMID4_SIZE, mmGDS_GWS_VMID4, mmGDS_OA_VMID4},
 184         {mmGDS_VMID5_BASE, mmGDS_VMID5_SIZE, mmGDS_GWS_VMID5, mmGDS_OA_VMID5},
 185         {mmGDS_VMID6_BASE, mmGDS_VMID6_SIZE, mmGDS_GWS_VMID6, mmGDS_OA_VMID6},
 186         {mmGDS_VMID7_BASE, mmGDS_VMID7_SIZE, mmGDS_GWS_VMID7, mmGDS_OA_VMID7},
 187         {mmGDS_VMID8_BASE, mmGDS_VMID8_SIZE, mmGDS_GWS_VMID8, mmGDS_OA_VMID8},
 188         {mmGDS_VMID9_BASE, mmGDS_VMID9_SIZE, mmGDS_GWS_VMID9, mmGDS_OA_VMID9},
 189         {mmGDS_VMID10_BASE, mmGDS_VMID10_SIZE, mmGDS_GWS_VMID10, mmGDS_OA_VMID10},
 190         {mmGDS_VMID11_BASE, mmGDS_VMID11_SIZE, mmGDS_GWS_VMID11, mmGDS_OA_VMID11},
 191         {mmGDS_VMID12_BASE, mmGDS_VMID12_SIZE, mmGDS_GWS_VMID12, mmGDS_OA_VMID12},
 192         {mmGDS_VMID13_BASE, mmGDS_VMID13_SIZE, mmGDS_GWS_VMID13, mmGDS_OA_VMID13},
 193         {mmGDS_VMID14_BASE, mmGDS_VMID14_SIZE, mmGDS_GWS_VMID14, mmGDS_OA_VMID14},
 194         {mmGDS_VMID15_BASE, mmGDS_VMID15_SIZE, mmGDS_GWS_VMID15, mmGDS_OA_VMID15}
 195 };
 196
 197 static const u32 golden_settings_tonga_a11[] =
 198 {
 199         mmCB_HW_CONTROL, 0xfffdf3cf, 0x00007208,
 200         mmCB_HW_CONTROL_3, 0x00000040, 0x00000040,
 201         mmDB_DEBUG2, 0xf00fffff, 0x00000400,
 202         mmGB_GPU_ID, 0x0000000f, 0x00000000,
 203         mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
 204         mmPA_SC_FIFO_DEPTH_CNTL, 0x000003ff, 0x000000fc,
 205         mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
 206         mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0000003c,
 207         mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,
 208         mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
 209         mmTCC_CTRL, 0x00100000, 0xf31fff7f,
 210         mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
 211         mmTCP_ADDR_CONFIG, 0x000003ff, 0x000002fb,
 212         mmTCP_CHAN_STEER_HI, 0xffffffff, 0x0000543b,
 213         mmTCP_CHAN_STEER_LO, 0xffffffff, 0xa9210876,
 214         mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,
 215 };
 216
 217 static const u32 tonga_golden_common_all[] =
 218 {
 219         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 220         mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x16000012,
 221         mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x0000002A,
 222         mmGB_ADDR_CONFIG, 0xffffffff, 0x22011003,
 223         mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
 224         mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
 225         mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF,
 226         mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF
 227 };
 228
 229 static const u32 tonga_mgcg_cgcg_init[] =
 230 {
 231         mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xffffffff,
 232         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 233         mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 234         mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
 235         mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
 236         mmCGTT_CPC_CLK_CTRL, 0xffffffff, 0x00000100,
 237         mmCGTT_CPF_CLK_CTRL, 0xffffffff, 0x40000100,
 238         mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
 239         mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
 240         mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
 241         mmCGTT_WD_CLK_CTRL, 0xffffffff, 0x06000100,
 242         mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
 243         mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
 244         mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
 245         mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
 246         mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
 247         mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
 248         mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
 249         mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
 250         mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
 251         mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
 252         mmCGTT_SX_CLK_CTRL4, 0xffffffff, 0x00000100,
 253         mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
 254         mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
 255         mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
 256         mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
 257         mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
 258         mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 259         mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 260         mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
 261         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 262         mmCGTS_CU0_SP0_CTRL_REG, 0xffffffff, 0x00010000,
 263         mmCGTS_CU0_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
 264         mmCGTS_CU0_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007,
 265         mmCGTS_CU0_SP1_CTRL_REG, 0xffffffff, 0x00060005,
 266         mmCGTS_CU0_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
 267         mmCGTS_CU1_SP0_CTRL_REG, 0xffffffff, 0x00010000,
 268         mmCGTS_CU1_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
 269         mmCGTS_CU1_TA_CTRL_REG, 0xffffffff, 0x00040007,
 270         mmCGTS_CU1_SP1_CTRL_REG, 0xffffffff, 0x00060005,
 271         mmCGTS_CU1_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
 272         mmCGTS_CU2_SP0_CTRL_REG, 0xffffffff, 0x00010000,
 273         mmCGTS_CU2_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
 274         mmCGTS_CU2_TA_CTRL_REG, 0xffffffff, 0x00040007,
 275         mmCGTS_CU2_SP1_CTRL_REG, 0xffffffff, 0x00060005,
 276         mmCGTS_CU2_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
 277         mmCGTS_CU3_SP0_CTRL_REG, 0xffffffff, 0x00010000,
 278         mmCGTS_CU3_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
 279         mmCGTS_CU3_TA_CTRL_REG, 0xffffffff, 0x00040007,
 280         mmCGTS_CU3_SP1_CTRL_REG, 0xffffffff, 0x00060005,
 281         mmCGTS_CU3_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
 282         mmCGTS_CU4_SP0_CTRL_REG, 0xffffffff, 0x00010000,
 283         mmCGTS_CU4_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
 284         mmCGTS_CU4_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007,
 285         mmCGTS_CU4_SP1_CTRL_REG, 0xffffffff, 0x00060005,
 286         mmCGTS_CU4_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
 287         mmCGTS_CU5_SP0_CTRL_REG, 0xffffffff, 0x00010000,
 288         mmCGTS_CU5_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
 289         mmCGTS_CU5_TA_CTRL_REG, 0xffffffff, 0x00040007,
 290         mmCGTS_CU5_SP1_CTRL_REG, 0xffffffff, 0x00060005,
 291         mmCGTS_CU5_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
 292         mmCGTS_CU6_SP0_CTRL_REG, 0xffffffff, 0x00010000,
 293         mmCGTS_CU6_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
 294         mmCGTS_CU6_TA_CTRL_REG, 0xffffffff, 0x00040007,
 295         mmCGTS_CU6_SP1_CTRL_REG, 0xffffffff, 0x00060005,
 296         mmCGTS_CU6_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
 297         mmCGTS_CU7_SP0_CTRL_REG, 0xffffffff, 0x00010000,
 298         mmCGTS_CU7_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
 299         mmCGTS_CU7_TA_CTRL_REG, 0xffffffff, 0x00040007,
 300         mmCGTS_CU7_SP1_CTRL_REG, 0xffffffff, 0x00060005,
 301         mmCGTS_CU7_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
 302         mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96e00200,
 303         mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
 304         mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003c,
 305         mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
 306 };
 307
 308 static const u32 golden_settings_vegam_a11[] =
 309 {
 310         mmCB_HW_CONTROL, 0x0001f3cf, 0x00007208,
 311         mmCB_HW_CONTROL_2, 0x0f000000, 0x0d000000,
 312         mmCB_HW_CONTROL_3, 0x000001ff, 0x00000040,
 313         mmDB_DEBUG2, 0xf00fffff, 0x00000400,
 314         mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
 315         mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
 316         mmPA_SC_RASTER_CONFIG, 0x3f3fffff, 0x3a00161a,
 317         mmPA_SC_RASTER_CONFIG_1, 0x0000003f, 0x0000002e,
 318         mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0001003c,
 319         mmRLC_CGCG_CGLS_CTRL_3D, 0xffffffff, 0x0001003c,
 320         mmSQ_CONFIG, 0x07f80000, 0x01180000,
 321         mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
 322         mmTCC_CTRL, 0x00100000, 0xf31fff7f,
 323         mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f7,
 324         mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,
 325         mmTCP_CHAN_STEER_LO, 0xffffffff, 0x32761054,
 326         mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,
 327 };
 328
 329 static const u32 vegam_golden_common_all[] =
 330 {
 331         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 332         mmGB_ADDR_CONFIG, 0xffffffff, 0x22011003,
 333         mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
 334         mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
 335         mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF,
 336         mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF,
 337 };
 338
 339 static const u32 golden_settings_polaris11_a11[] =
 340 {
 341         mmCB_HW_CONTROL, 0x0000f3cf, 0x00007208,
 342         mmCB_HW_CONTROL_2, 0x0f000000, 0x0f000000,
 343         mmCB_HW_CONTROL_3, 0x000001ff, 0x00000040,
 344         mmDB_DEBUG2, 0xf00fffff, 0x00000400,
 345         mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
 346         mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
 347         mmPA_SC_RASTER_CONFIG, 0x3f3fffff, 0x16000012,
 348         mmPA_SC_RASTER_CONFIG_1, 0x0000003f, 0x00000000,
 349         mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0001003c,
 350         mmRLC_CGCG_CGLS_CTRL_3D, 0xffffffff, 0x0001003c,
 351         mmSQ_CONFIG, 0x07f80000, 0x01180000,
 352         mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
 353         mmTCC_CTRL, 0x00100000, 0xf31fff7f,
 354         mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f3,
 355         mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,
 356         mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003210,
 357         mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,
 358 };
 359
 360 static const u32 polaris11_golden_common_all[] =
 361 {
 362         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 363         mmGB_ADDR_CONFIG, 0xffffffff, 0x22011002,
 364         mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
 365         mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
 366         mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF,
 367         mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF,
 368 };
 369
 370 static const u32 golden_settings_polaris10_a11[] =
 371 {
 372         mmATC_MISC_CG, 0x000c0fc0, 0x000c0200,
 373         mmCB_HW_CONTROL, 0x0001f3cf, 0x00007208,
 374         mmCB_HW_CONTROL_2, 0x0f000000, 0x0f000000,
 375         mmCB_HW_CONTROL_3, 0x000001ff, 0x00000040,
 376         mmDB_DEBUG2, 0xf00fffff, 0x00000400,
 377         mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
 378         mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
 379         mmPA_SC_RASTER_CONFIG, 0x3f3fffff, 0x16000012,
 380         mmPA_SC_RASTER_CONFIG_1, 0x0000003f, 0x0000002a,
 381         mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0001003c,
 382         mmRLC_CGCG_CGLS_CTRL_3D, 0xffffffff, 0x0001003c,
 383         mmSQ_CONFIG, 0x07f80000, 0x07180000,
 384         mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
 385         mmTCC_CTRL, 0x00100000, 0xf31fff7f,
 386         mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f7,
 387         mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,
 388         mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,
 389 };
 390
 391 static const u32 polaris10_golden_common_all[] =
 392 {
 393         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 394         mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x16000012,
 395         mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x0000002A,
 396         mmGB_ADDR_CONFIG, 0xffffffff, 0x22011003,
 397         mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
 398         mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
 399         mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF,
 400         mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF,
 401 };
 402
 403 static const u32 fiji_golden_common_all[] =
 404 {
 405         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 406         mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x3a00161a,
 407         mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x0000002e,
 408         mmGB_ADDR_CONFIG, 0xffffffff, 0x22011003,
 409         mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
 410         mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
 411         mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF,
 412         mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF,
 413         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 414         mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x00000009,
 415 };
 416
 417 static const u32 golden_settings_fiji_a10[] =
 418 {
 419         mmCB_HW_CONTROL_3, 0x000001ff, 0x00000040,
 420         mmDB_DEBUG2, 0xf00fffff, 0x00000400,
 421         mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
 422         mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
 423         mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0001003c,
 424         mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,
 425         mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
 426         mmTCC_CTRL, 0x00100000, 0xf31fff7f,
 427         mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
 428         mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000ff,
 429         mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,
 430 };
 431
 432 static const u32 fiji_mgcg_cgcg_init[] =
 433 {
 434         mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xffffffff,
 435         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 436         mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 437         mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
 438         mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
 439         mmCGTT_CPC_CLK_CTRL, 0xffffffff, 0x00000100,
 440         mmCGTT_CPF_CLK_CTRL, 0xffffffff, 0x40000100,
 441         mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
 442         mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
 443         mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
 444         mmCGTT_WD_CLK_CTRL, 0xffffffff, 0x06000100,
 445         mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
 446         mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
 447         mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
 448         mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
 449         mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
 450         mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
 451         mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
 452         mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
 453         mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
 454         mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
 455         mmCGTT_SX_CLK_CTRL4, 0xffffffff, 0x00000100,
 456         mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
 457         mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
 458         mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
 459         mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
 460         mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
 461         mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 462         mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 463         mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
 464         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 465         mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96e00200,
 466         mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
 467         mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003c,
 468         mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
 469 };
 470
 471 static const u32 golden_settings_iceland_a11[] =
 472 {
 473         mmCB_HW_CONTROL_3, 0x00000040, 0x00000040,
 474         mmDB_DEBUG2, 0xf00fffff, 0x00000400,
 475         mmDB_DEBUG3, 0xc0000000, 0xc0000000,
 476         mmGB_GPU_ID, 0x0000000f, 0x00000000,
 477         mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
 478         mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
 479         mmPA_SC_RASTER_CONFIG, 0x3f3fffff, 0x00000002,
 480         mmPA_SC_RASTER_CONFIG_1, 0x0000003f, 0x00000000,
 481         mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0000003c,
 482         mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,
 483         mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
 484         mmTCC_CTRL, 0x00100000, 0xf31fff7f,
 485         mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
 486         mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f1,
 487         mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,
 488         mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00000010,
 489 };
 490
 491 static const u32 iceland_golden_common_all[] =
 492 {
 493         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 494         mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x00000002,
 495         mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x00000000,
 496         mmGB_ADDR_CONFIG, 0xffffffff, 0x22010001,
 497         mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
 498         mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
 499         mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF,
 500         mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF
 501 };
 502
 503 static const u32 iceland_mgcg_cgcg_init[] =
 504 {
 505         mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xffffffff,
 506         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 507         mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 508         mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
 509         mmCGTT_CP_CLK_CTRL, 0xffffffff, 0xc0000100,
 510         mmCGTT_CPC_CLK_CTRL, 0xffffffff, 0xc0000100,
 511         mmCGTT_CPF_CLK_CTRL, 0xffffffff, 0xc0000100,
 512         mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
 513         mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
 514         mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
 515         mmCGTT_WD_CLK_CTRL, 0xffffffff, 0x06000100,
 516         mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
 517         mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
 518         mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
 519         mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
 520         mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
 521         mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
 522         mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
 523         mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
 524         mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
 525         mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
 526         mmCGTT_SX_CLK_CTRL4, 0xffffffff, 0x00000100,
 527         mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0xff000100,
 528         mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
 529         mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
 530         mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
 531         mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
 532         mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 533         mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 534         mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
 535         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 536         mmCGTS_CU0_SP0_CTRL_REG, 0xffffffff, 0x00010000,
 537         mmCGTS_CU0_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
 538         mmCGTS_CU0_TA_SQC_CTRL_REG, 0xffffffff, 0x0f840f87,
 539         mmCGTS_CU0_SP1_CTRL_REG, 0xffffffff, 0x00060005,
 540         mmCGTS_CU0_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
 541         mmCGTS_CU1_SP0_CTRL_REG, 0xffffffff, 0x00010000,
 542         mmCGTS_CU1_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
 543         mmCGTS_CU1_TA_CTRL_REG, 0xffffffff, 0x00040007,
 544         mmCGTS_CU1_SP1_CTRL_REG, 0xffffffff, 0x00060005,
 545         mmCGTS_CU1_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
 546         mmCGTS_CU2_SP0_CTRL_REG, 0xffffffff, 0x00010000,
 547         mmCGTS_CU2_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
 548         mmCGTS_CU2_TA_CTRL_REG, 0xffffffff, 0x00040007,
 549         mmCGTS_CU2_SP1_CTRL_REG, 0xffffffff, 0x00060005,
 550         mmCGTS_CU2_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
 551         mmCGTS_CU3_SP0_CTRL_REG, 0xffffffff, 0x00010000,
 552         mmCGTS_CU3_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
 553         mmCGTS_CU3_TA_CTRL_REG, 0xffffffff, 0x00040007,
 554         mmCGTS_CU3_SP1_CTRL_REG, 0xffffffff, 0x00060005,
 555         mmCGTS_CU3_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
 556         mmCGTS_CU4_SP0_CTRL_REG, 0xffffffff, 0x00010000,
 557         mmCGTS_CU4_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
 558         mmCGTS_CU4_TA_SQC_CTRL_REG, 0xffffffff, 0x0f840f87,
 559         mmCGTS_CU4_SP1_CTRL_REG, 0xffffffff, 0x00060005,
 560         mmCGTS_CU4_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
 561         mmCGTS_CU5_SP0_CTRL_REG, 0xffffffff, 0x00010000,
 562         mmCGTS_CU5_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
 563         mmCGTS_CU5_TA_CTRL_REG, 0xffffffff, 0x00040007,
 564         mmCGTS_CU5_SP1_CTRL_REG, 0xffffffff, 0x00060005,
 565         mmCGTS_CU5_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
 566         mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96e00200,
 567         mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
 568         mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003c,
 569 };
 570
 571 static const u32 cz_golden_settings_a11[] =
 572 {
 573         mmCB_HW_CONTROL_3, 0x00000040, 0x00000040,
 574         mmDB_DEBUG2, 0xf00fffff, 0x00000400,
 575         mmGB_GPU_ID, 0x0000000f, 0x00000000,
 576         mmPA_SC_ENHANCE, 0xffffffff, 0x00000001,
 577         mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
 578         mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0000003c,
 579         mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,
 580         mmTA_CNTL_AUX, 0x000f000f, 0x00010000,
 581         mmTCC_CTRL, 0x00100000, 0xf31fff7f,
 582         mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
 583         mmTCP_ADDR_CONFIG, 0x0000000f, 0x000000f3,
 584         mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00001302
 585 };
 586
 587 static const u32 cz_golden_common_all[] =
 588 {
 589         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 590         mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x00000002,
 591         mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x00000000,
 592         mmGB_ADDR_CONFIG, 0xffffffff, 0x22010001,
 593         mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
 594         mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
 595         mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF,
 596         mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF
 597 };
 598
 599 static const u32 cz_mgcg_cgcg_init[] =
 600 {
 601         mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xffffffff,
 602         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 603         mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 604         mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
 605         mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
 606         mmCGTT_CPC_CLK_CTRL, 0xffffffff, 0x00000100,
 607         mmCGTT_CPF_CLK_CTRL, 0xffffffff, 0x00000100,
 608         mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
 609         mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
 610         mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
 611         mmCGTT_WD_CLK_CTRL, 0xffffffff, 0x06000100,
 612         mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
 613         mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
 614         mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
 615         mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
 616         mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
 617         mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
 618         mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
 619         mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
 620         mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
 621         mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
 622         mmCGTT_SX_CLK_CTRL4, 0xffffffff, 0x00000100,
 623         mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
 624         mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
 625         mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
 626         mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
 627         mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
 628         mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 629         mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 630         mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
 631         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 632         mmCGTS_CU0_SP0_CTRL_REG, 0xffffffff, 0x00010000,
 633         mmCGTS_CU0_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
 634         mmCGTS_CU0_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007,
 635         mmCGTS_CU0_SP1_CTRL_REG, 0xffffffff, 0x00060005,
 636         mmCGTS_CU0_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
 637         mmCGTS_CU1_SP0_CTRL_REG, 0xffffffff, 0x00010000,
 638         mmCGTS_CU1_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
 639         mmCGTS_CU1_TA_CTRL_REG, 0xffffffff, 0x00040007,
 640         mmCGTS_CU1_SP1_CTRL_REG, 0xffffffff, 0x00060005,
 641         mmCGTS_CU1_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
 642         mmCGTS_CU2_SP0_CTRL_REG, 0xffffffff, 0x00010000,
 643         mmCGTS_CU2_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
 644         mmCGTS_CU2_TA_CTRL_REG, 0xffffffff, 0x00040007,
 645         mmCGTS_CU2_SP1_CTRL_REG, 0xffffffff, 0x00060005,
 646         mmCGTS_CU2_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
 647         mmCGTS_CU3_SP0_CTRL_REG, 0xffffffff, 0x00010000,
 648         mmCGTS_CU3_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
 649         mmCGTS_CU3_TA_CTRL_REG, 0xffffffff, 0x00040007,
 650         mmCGTS_CU3_SP1_CTRL_REG, 0xffffffff, 0x00060005,
 651         mmCGTS_CU3_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
 652         mmCGTS_CU4_SP0_CTRL_REG, 0xffffffff, 0x00010000,
 653         mmCGTS_CU4_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
 654         mmCGTS_CU4_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007,
 655         mmCGTS_CU4_SP1_CTRL_REG, 0xffffffff, 0x00060005,
 656         mmCGTS_CU4_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
 657         mmCGTS_CU5_SP0_CTRL_REG, 0xffffffff, 0x00010000,
 658         mmCGTS_CU5_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
 659         mmCGTS_CU5_TA_CTRL_REG, 0xffffffff, 0x00040007,
 660         mmCGTS_CU5_SP1_CTRL_REG, 0xffffffff, 0x00060005,
 661         mmCGTS_CU5_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
 662         mmCGTS_CU6_SP0_CTRL_REG, 0xffffffff, 0x00010000,
 663         mmCGTS_CU6_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
 664         mmCGTS_CU6_TA_CTRL_REG, 0xffffffff, 0x00040007,
 665         mmCGTS_CU6_SP1_CTRL_REG, 0xffffffff, 0x00060005,
 666         mmCGTS_CU6_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
 667         mmCGTS_CU7_SP0_CTRL_REG, 0xffffffff, 0x00010000,
 668         mmCGTS_CU7_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
 669         mmCGTS_CU7_TA_CTRL_REG, 0xffffffff, 0x00040007,
 670         mmCGTS_CU7_SP1_CTRL_REG, 0xffffffff, 0x00060005,
 671         mmCGTS_CU7_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
 672         mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96e00200,
 673         mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
 674         mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,
 675         mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
 676 };
 677
 678 static const u32 stoney_golden_settings_a11[] =
 679 {
 680         mmDB_DEBUG2, 0xf00fffff, 0x00000400,
 681         mmGB_GPU_ID, 0x0000000f, 0x00000000,
 682         mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
 683         mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
 684         mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0001003c,
 685         mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
 686         mmTCC_CTRL, 0x00100000, 0xf31fff7f,
 687         mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
 688         mmTCP_ADDR_CONFIG, 0x0000000f, 0x000000f1,
 689         mmTCP_CHAN_STEER_LO, 0xffffffff, 0x10101010,
 690 };
 691
 692 static const u32 stoney_golden_common_all[] =
 693 {
 694         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 695         mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x00000000,
 696         mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x00000000,
 697         mmGB_ADDR_CONFIG, 0xffffffff, 0x12010001,
 698         mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
 699         mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
 700         mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF,
 701         mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF,
 702 };
 703
 704 static const u32 stoney_mgcg_cgcg_init[] =
 705 {
 706         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 707         mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,
 708         mmCP_MEM_SLP_CNTL, 0xffffffff, 0x00020201,
 709         mmRLC_MEM_SLP_CNTL, 0xffffffff, 0x00020201,
 710         mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,
 711 };
 712
 713
 714 static const char * const sq_edc_source_names[] = {
 715         "SQ_EDC_INFO_SOURCE_INVALID: No EDC error has occurred",
 716         "SQ_EDC_INFO_SOURCE_INST: EDC source is Instruction Fetch",
 717         "SQ_EDC_INFO_SOURCE_SGPR: EDC source is SGPR or SQC data return",
 718         "SQ_EDC_INFO_SOURCE_VGPR: EDC source is VGPR",
 719         "SQ_EDC_INFO_SOURCE_LDS: EDC source is LDS",
 720         "SQ_EDC_INFO_SOURCE_GDS: EDC source is GDS",
 721         "SQ_EDC_INFO_SOURCE_TA: EDC source is TA",
 722 };
 723
 724 static void gfx_v8_0_set_ring_funcs(struct amdgpu_device *adev);
 725 static void gfx_v8_0_set_irq_funcs(struct amdgpu_device *adev);
 726 static void gfx_v8_0_set_gds_init(struct amdgpu_device *adev);
 727 static void gfx_v8_0_set_rlc_funcs(struct amdgpu_device *adev);
 728 static u32 gfx_v8_0_get_csb_size(struct amdgpu_device *adev);
 729 static void gfx_v8_0_get_cu_info(struct amdgpu_device *adev);
 730 static void gfx_v8_0_ring_emit_ce_meta(struct amdgpu_ring *ring);
 731 static void gfx_v8_0_ring_emit_de_meta(struct amdgpu_ring *ring);
 732
 733 #define CG_ACLK_CNTL__ACLK_DIVIDER_MASK                    0x0000007fL
 734 #define CG_ACLK_CNTL__ACLK_DIVIDER__SHIFT                  0x00000000L
 735
 736 static void gfx_v8_0_init_golden_registers(struct amdgpu_device *adev)
 737 {
 738         uint32_t data;
 739
 740         switch (adev->asic_type) {
 741         case CHIP_TOPAZ:
 742                 amdgpu_device_program_register_sequence(adev,
 743                                                         iceland_mgcg_cgcg_init,
 744                                                         ARRAY_SIZE(iceland_mgcg_cgcg_init));
 745                 amdgpu_device_program_register_sequence(adev,
 746                                                         golden_settings_iceland_a11,
 747                                                         ARRAY_SIZE(golden_settings_iceland_a11));
 748                 amdgpu_device_program_register_sequence(adev,
 749                                                         iceland_golden_common_all,
 750                                                         ARRAY_SIZE(iceland_golden_common_all));
 751                 break;
 752         case CHIP_FIJI:
 753                 amdgpu_device_program_register_sequence(adev,
 754                                                         fiji_mgcg_cgcg_init,
 755                                                         ARRAY_SIZE(fiji_mgcg_cgcg_init));
 756                 amdgpu_device_program_register_sequence(adev,
 757                                                         golden_settings_fiji_a10,
 758                                                         ARRAY_SIZE(golden_settings_fiji_a10));
 759                 amdgpu_device_program_register_sequence(adev,
 760                                                         fiji_golden_common_all,
 761                                                         ARRAY_SIZE(fiji_golden_common_all));
 762                 break;
 763
 764         case CHIP_TONGA:
 765                 amdgpu_device_program_register_sequence(adev,
 766                                                         tonga_mgcg_cgcg_init,
 767                                                         ARRAY_SIZE(tonga_mgcg_cgcg_init));
 768                 amdgpu_device_program_register_sequence(adev,
 769                                                         golden_settings_tonga_a11,
 770                                                         ARRAY_SIZE(golden_settings_tonga_a11));
 771                 amdgpu_device_program_register_sequence(adev,
 772                                                         tonga_golden_common_all,
 773                                                         ARRAY_SIZE(tonga_golden_common_all));
 774                 break;
 775         case CHIP_VEGAM:
 776                 amdgpu_device_program_register_sequence(adev,
 777                                                         golden_settings_vegam_a11,
 778                                                         ARRAY_SIZE(golden_settings_vegam_a11));
 779                 amdgpu_device_program_register_sequence(adev,
 780                                                         vegam_golden_common_all,
 781                                                         ARRAY_SIZE(vegam_golden_common_all));
 782                 break;
 783         case CHIP_POLARIS11:
 784         case CHIP_POLARIS12:
 785                 amdgpu_device_program_register_sequence(adev,
 786                                                         golden_settings_polaris11_a11,
 787                                                         ARRAY_SIZE(golden_settings_polaris11_a11));
 788                 amdgpu_device_program_register_sequence(adev,
 789                                                         polaris11_golden_common_all,
 790                                                         ARRAY_SIZE(polaris11_golden_common_all));
 791                 break;
 792         case CHIP_POLARIS10:
 793                 amdgpu_device_program_register_sequence(adev,
 794                                                         golden_settings_polaris10_a11,
 795                                                         ARRAY_SIZE(golden_settings_polaris10_a11));
 796                 amdgpu_device_program_register_sequence(adev,
 797                                                         polaris10_golden_common_all,
 798                                                         ARRAY_SIZE(polaris10_golden_common_all));
 799                 data = RREG32_SMC(ixCG_ACLK_CNTL);
 800                 data &= ~CG_ACLK_CNTL__ACLK_DIVIDER_MASK;
 801                 data |= 0x18 << CG_ACLK_CNTL__ACLK_DIVIDER__SHIFT;
 802                 WREG32_SMC(ixCG_ACLK_CNTL, data);
 803                 if ((adev->pdev->device == 0x67DF) && (adev->pdev->revision == 0xc7) &&
 804                     ((adev->pdev->subsystem_device == 0xb37 && adev->pdev->subsystem_vendor == 0x1002) ||
 805                      (adev->pdev->subsystem_device == 0x4a8 && adev->pdev->subsystem_vendor == 0x1043) ||
 806                      (adev->pdev->subsystem_device == 0x9480 && adev->pdev->subsystem_vendor == 0x1680))) {
 807                         amdgpu_atombios_i2c_channel_trans(adev, 0x10, 0x96, 0x1E, 0xDD);
 808                         amdgpu_atombios_i2c_channel_trans(adev, 0x10, 0x96, 0x1F, 0xD0);
 809                 }
 810                 break;
 811         case CHIP_CARRIZO:
 812                 amdgpu_device_program_register_sequence(adev,
 813                                                         cz_mgcg_cgcg_init,
 814                                                         ARRAY_SIZE(cz_mgcg_cgcg_init));
 815                 amdgpu_device_program_register_sequence(adev,
 816                                                         cz_golden_settings_a11,
 817                                                         ARRAY_SIZE(cz_golden_settings_a11));
 818                 amdgpu_device_program_register_sequence(adev,
 819                                                         cz_golden_common_all,
 820                                                         ARRAY_SIZE(cz_golden_common_all));
 821                 break;
 822         case CHIP_STONEY:
 823                 amdgpu_device_program_register_sequence(adev,
 824                                                         stoney_mgcg_cgcg_init,
 825                                                         ARRAY_SIZE(stoney_mgcg_cgcg_init));
 826                 amdgpu_device_program_register_sequence(adev,
 827                                                         stoney_golden_settings_a11,
 828                                                         ARRAY_SIZE(stoney_golden_settings_a11));
 829                 amdgpu_device_program_register_sequence(adev,
 830                                                         stoney_golden_common_all,
 831                                                         ARRAY_SIZE(stoney_golden_common_all));
 832                 break;
 833         default:
 834                 break;
 835         }
 836 }
 837
 838 static int gfx_v8_0_ring_test_ring(struct amdgpu_ring *ring)
 839 {
 840         struct amdgpu_device *adev = ring->adev;
 841         uint32_t tmp = 0;
 842         unsigned i;
 843         int r;
 844
 845         WREG32(mmSCRATCH_REG0, 0xCAFEDEAD);
 846         r = amdgpu_ring_alloc(ring, 3);
 847         if (r)
 848                 return r;
 849
 850         amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
 851         amdgpu_ring_write(ring, mmSCRATCH_REG0 - PACKET3_SET_UCONFIG_REG_START);
 852         amdgpu_ring_write(ring, 0xDEADBEEF);
 853         amdgpu_ring_commit(ring);
 854
 855         for (i = 0; i < adev->usec_timeout; i++) {
 856                 tmp = RREG32(mmSCRATCH_REG0);
 857                 if (tmp == 0xDEADBEEF)
 858                         break;
 859                 udelay(1);
 860         }
 861
 862         if (i >= adev->usec_timeout)
 863                 r = -ETIMEDOUT;
 864
 865         return r;
 866 }
 867
 868 static int gfx_v8_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 869 {
 870         struct amdgpu_device *adev = ring->adev;
 871         struct amdgpu_ib ib;
 872         struct dma_fence *f = NULL;
 873
 874         unsigned int index;
 875         uint64_t gpu_addr;
 876         uint32_t tmp;
 877         long r;
 878
 879         r = amdgpu_device_wb_get(adev, &index);
 880         if (r)
 881                 return r;
 882
 883         gpu_addr = adev->wb.gpu_addr + (index * 4);
 884         adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD);
 885         memset(&ib, 0, sizeof(ib));
 886         r = amdgpu_ib_get(adev, NULL, 16,
 887                                         AMDGPU_IB_POOL_DIRECT, &ib);
 888         if (r)
 889                 goto err1;
 890
 891         ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3);
 892         ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM;
 893         ib.ptr[2] = lower_32_bits(gpu_addr);
 894         ib.ptr[3] = upper_32_bits(gpu_addr);
 895         ib.ptr[4] = 0xDEADBEEF;
 896         ib.length_dw = 5;
 897
 898         r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
 899         if (r)
 900                 goto err2;
 901
 902         r = dma_fence_wait_timeout(f, false, timeout);
 903         if (r == 0) {
 904                 r = -ETIMEDOUT;
 905                 goto err2;
 906         } else if (r < 0) {
 907                 goto err2;
 908         }
 909
 910         tmp = adev->wb.wb[index];
 911         if (tmp == 0xDEADBEEF)
 912                 r = 0;
 913         else
 914                 r = -EINVAL;
 915
 916 err2:
 917         amdgpu_ib_free(adev, &ib, NULL);
 918         dma_fence_put(f);
 919 err1:
 920         amdgpu_device_wb_free(adev, index);
 921         return r;
 922 }
 923
 924
 925 static void gfx_v8_0_free_microcode(struct amdgpu_device *adev)
 926 {
 927         amdgpu_ucode_release(&adev->gfx.pfp_fw);
 928         amdgpu_ucode_release(&adev->gfx.me_fw);
 929         amdgpu_ucode_release(&adev->gfx.ce_fw);
 930         amdgpu_ucode_release(&adev->gfx.rlc_fw);
 931         amdgpu_ucode_release(&adev->gfx.mec_fw);
 932         if ((adev->asic_type != CHIP_STONEY) &&
 933             (adev->asic_type != CHIP_TOPAZ))
 934                 amdgpu_ucode_release(&adev->gfx.mec2_fw);
 935
 936         kfree(adev->gfx.rlc.register_list_format);
 937 }
 938
 939 static int gfx_v8_0_init_microcode(struct amdgpu_device *adev)
 940 {
 941         const char *chip_name;
 942         char fw_name[30];
 943         int err;
 944         struct amdgpu_firmware_info *info = NULL;
 945         const struct common_firmware_header *header = NULL;
 946         const struct gfx_firmware_header_v1_0 *cp_hdr;
 947         const struct rlc_firmware_header_v2_0 *rlc_hdr;
 948         unsigned int *tmp = NULL, i;
 949
 950         DRM_DEBUG("\n");
 951
 952         switch (adev->asic_type) {
 953         case CHIP_TOPAZ:
 954                 chip_name = "topaz";
 955                 break;
 956         case CHIP_TONGA:
 957                 chip_name = "tonga";
 958                 break;
 959         case CHIP_CARRIZO:
 960                 chip_name = "carrizo";
 961                 break;
 962         case CHIP_FIJI:
 963                 chip_name = "fiji";
 964                 break;
 965         case CHIP_STONEY:
 966                 chip_name = "stoney";
 967                 break;
 968         case CHIP_POLARIS10:
 969                 chip_name = "polaris10";
 970                 break;
 971         case CHIP_POLARIS11:
 972                 chip_name = "polaris11";
 973                 break;
 974         case CHIP_POLARIS12:
 975                 chip_name = "polaris12";
 976                 break;
 977         case CHIP_VEGAM:
 978                 chip_name = "vegam";
 979                 break;
 980         default:
 981                 BUG();
 982         }
 983
 984         if (adev->asic_type >= CHIP_POLARIS10 && adev->asic_type <= CHIP_POLARIS12) {
 985                 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_pfp_2.bin", chip_name);
 986                 err = amdgpu_ucode_request(adev, &adev->gfx.pfp_fw, fw_name);
 987                 if (err == -ENODEV) {
 988                         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_pfp.bin", chip_name);
 989                         err = amdgpu_ucode_request(adev, &adev->gfx.pfp_fw, fw_name);
 990                 }
 991         } else {
 992                 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_pfp.bin", chip_name);
 993                 err = amdgpu_ucode_request(adev, &adev->gfx.pfp_fw, fw_name);
 994         }
 995         if (err)
 996                 goto out;
 997         cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.pfp_fw->data;
 998         adev->gfx.pfp_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
 999         adev->gfx.pfp_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
1000
1001         if (adev->asic_type >= CHIP_POLARIS10 && adev->asic_type <= CHIP_POLARIS12) {
1002                 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_me_2.bin", chip_name);
1003                 err = amdgpu_ucode_request(adev, &adev->gfx.me_fw, fw_name);
1004                 if (err == -ENODEV) {
1005                         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_me.bin", chip_name);
1006                         err = amdgpu_ucode_request(adev, &adev->gfx.me_fw, fw_name);
1007                 }
1008         } else {
1009                 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_me.bin", chip_name);
1010                 err = amdgpu_ucode_request(adev, &adev->gfx.me_fw, fw_name);
1011         }
1012         if (err)
1013                 goto out;
1014         cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.me_fw->data;
1015         adev->gfx.me_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
1016
1017         adev->gfx.me_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
1018
1019         if (adev->asic_type >= CHIP_POLARIS10 && adev->asic_type <= CHIP_POLARIS12) {
1020                 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_ce_2.bin", chip_name);
1021                 err = amdgpu_ucode_request(adev, &adev->gfx.ce_fw, fw_name);
1022                 if (err == -ENODEV) {
1023                         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_ce.bin", chip_name);
1024                         err = amdgpu_ucode_request(adev, &adev->gfx.ce_fw, fw_name);
1025                 }
1026         } else {
1027                 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_ce.bin", chip_name);
1028                 err = amdgpu_ucode_request(adev, &adev->gfx.ce_fw, fw_name);
1029         }
1030         if (err)
1031                 goto out;
1032         cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.ce_fw->data;
1033         adev->gfx.ce_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
1034         adev->gfx.ce_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
1035
1036         /*
1037          * Support for MCBP/Virtualization in combination with chained IBs is
1038          * formal released on feature version #46
1039          */
1040         if (adev->gfx.ce_feature_version >= 46 &&
1041             adev->gfx.pfp_feature_version >= 46) {
1042                 adev->virt.chained_ib_support = true;
1043                 DRM_INFO("Chained IB support enabled!\n");
1044         } else
1045                 adev->virt.chained_ib_support = false;
1046
1047         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_rlc.bin", chip_name);
1048         err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw, fw_name);
1049         if (err)
1050                 goto out;
1051         rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
1052         adev->gfx.rlc_fw_version = le32_to_cpu(rlc_hdr->header.ucode_version);
1053         adev->gfx.rlc_feature_version = le32_to_cpu(rlc_hdr->ucode_feature_version);
1054
1055         adev->gfx.rlc.save_and_restore_offset =
1056                         le32_to_cpu(rlc_hdr->save_and_restore_offset);
1057         adev->gfx.rlc.clear_state_descriptor_offset =
1058                         le32_to_cpu(rlc_hdr->clear_state_descriptor_offset);
1059         adev->gfx.rlc.avail_scratch_ram_locations =
1060                         le32_to_cpu(rlc_hdr->avail_scratch_ram_locations);
1061         adev->gfx.rlc.reg_restore_list_size =
1062                         le32_to_cpu(rlc_hdr->reg_restore_list_size);
1063         adev->gfx.rlc.reg_list_format_start =
1064                         le32_to_cpu(rlc_hdr->reg_list_format_start);
1065         adev->gfx.rlc.reg_list_format_separate_start =
1066                         le32_to_cpu(rlc_hdr->reg_list_format_separate_start);
1067         adev->gfx.rlc.starting_offsets_start =
1068                         le32_to_cpu(rlc_hdr->starting_offsets_start);
1069         adev->gfx.rlc.reg_list_format_size_bytes =
1070                         le32_to_cpu(rlc_hdr->reg_list_format_size_bytes);
1071         adev->gfx.rlc.reg_list_size_bytes =
1072                         le32_to_cpu(rlc_hdr->reg_list_size_bytes);
1073
1074         adev->gfx.rlc.register_list_format =
1075                         kmalloc(adev->gfx.rlc.reg_list_format_size_bytes +
1076                                         adev->gfx.rlc.reg_list_size_bytes, GFP_KERNEL);
1077
1078         if (!adev->gfx.rlc.register_list_format) {
1079                 err = -ENOMEM;
1080                 goto out;
1081         }
1082
1083         tmp = (unsigned int *)((uintptr_t)rlc_hdr +
1084                         le32_to_cpu(rlc_hdr->reg_list_format_array_offset_bytes));
1085         for (i = 0 ; i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2); i++)
1086                 adev->gfx.rlc.register_list_format[i] = le32_to_cpu(tmp[i]);
1087
1088         adev->gfx.rlc.register_restore = adev->gfx.rlc.register_list_format + i;
1089
1090         tmp = (unsigned int *)((uintptr_t)rlc_hdr +
1091                         le32_to_cpu(rlc_hdr->reg_list_array_offset_bytes));
1092         for (i = 0 ; i < (adev->gfx.rlc.reg_list_size_bytes >> 2); i++)
1093                 adev->gfx.rlc.register_restore[i] = le32_to_cpu(tmp[i]);
1094
1095         if (adev->asic_type >= CHIP_POLARIS10 && adev->asic_type <= CHIP_POLARIS12) {
1096                 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec_2.bin", chip_name);
1097                 err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw, fw_name);
1098                 if (err == -ENODEV) {
1099                         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec.bin", chip_name);
1100                         err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw, fw_name);
1101                 }
1102         } else {
1103                 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec.bin", chip_name);
1104                 err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw, fw_name);
1105         }
1106         if (err)
1107                 goto out;
1108         cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
1109         adev->gfx.mec_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
1110         adev->gfx.mec_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
1111
1112         if ((adev->asic_type != CHIP_STONEY) &&
1113             (adev->asic_type != CHIP_TOPAZ)) {
1114                 if (adev->asic_type >= CHIP_POLARIS10 && adev->asic_type <= CHIP_POLARIS12) {
1115                         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec2_2.bin", chip_name);
1116                         err = amdgpu_ucode_request(adev, &adev->gfx.mec2_fw, fw_name);
1117                         if (err == -ENODEV) {
1118                                 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec2.bin", chip_name);
1119                                 err = amdgpu_ucode_request(adev, &adev->gfx.mec2_fw, fw_name);
1120                         }
1121                 } else {
1122                         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec2.bin", chip_name);
1123                         err = amdgpu_ucode_request(adev, &adev->gfx.mec2_fw, fw_name);
1124                 }
1125                 if (!err) {
1126                         cp_hdr = (const struct gfx_firmware_header_v1_0 *)
1127                                 adev->gfx.mec2_fw->data;
1128                         adev->gfx.mec2_fw_version =
1129                                 le32_to_cpu(cp_hdr->header.ucode_version);
1130                         adev->gfx.mec2_feature_version =
1131                                 le32_to_cpu(cp_hdr->ucode_feature_version);
1132                 } else {
1133                         err = 0;
1134                         adev->gfx.mec2_fw = NULL;
1135                 }
1136         }
1137
1138         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_PFP];
1139         info->ucode_id = AMDGPU_UCODE_ID_CP_PFP;
1140         info->fw = adev->gfx.pfp_fw;
1141         header = (const struct common_firmware_header *)info->fw->data;
1142         adev->firmware.fw_size +=
1143                 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1144
1145         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_ME];
1146         info->ucode_id = AMDGPU_UCODE_ID_CP_ME;
1147         info->fw = adev->gfx.me_fw;
1148         header = (const struct common_firmware_header *)info->fw->data;
1149         adev->firmware.fw_size +=
1150                 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1151
1152         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_CE];
1153         info->ucode_id = AMDGPU_UCODE_ID_CP_CE;
1154         info->fw = adev->gfx.ce_fw;
1155         header = (const struct common_firmware_header *)info->fw->data;
1156         adev->firmware.fw_size +=
1157                 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1158
1159         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_G];
1160         info->ucode_id = AMDGPU_UCODE_ID_RLC_G;
1161         info->fw = adev->gfx.rlc_fw;
1162         header = (const struct common_firmware_header *)info->fw->data;
1163         adev->firmware.fw_size +=
1164                 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1165
1166         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1];
1167         info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1;
1168         info->fw = adev->gfx.mec_fw;
1169         header = (const struct common_firmware_header *)info->fw->data;
1170         adev->firmware.fw_size +=
1171                 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1172
1173         /* we need account JT in */
1174         cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
1175         adev->firmware.fw_size +=
1176                 ALIGN(le32_to_cpu(cp_hdr->jt_size) << 2, PAGE_SIZE);
1177
1178         if (amdgpu_sriov_vf(adev)) {
1179                 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_STORAGE];
1180                 info->ucode_id = AMDGPU_UCODE_ID_STORAGE;
1181                 info->fw = adev->gfx.mec_fw;
1182                 adev->firmware.fw_size +=
1183                         ALIGN(le32_to_cpu(64 * PAGE_SIZE), PAGE_SIZE);
1184         }
1185
1186         if (adev->gfx.mec2_fw) {
1187                 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC2];
1188                 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC2;
1189                 info->fw = adev->gfx.mec2_fw;
1190                 header = (const struct common_firmware_header *)info->fw->data;
1191                 adev->firmware.fw_size +=
1192                         ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1193         }
1194
1195 out:
1196         if (err) {
1197                 dev_err(adev->dev,
1198                         "gfx8: Failed to load firmware \"%s\"\n",
1199                         fw_name);
1200                 amdgpu_ucode_release(&adev->gfx.pfp_fw);
1201                 amdgpu_ucode_release(&adev->gfx.me_fw);
1202                 amdgpu_ucode_release(&adev->gfx.ce_fw);
1203                 amdgpu_ucode_release(&adev->gfx.rlc_fw);
1204                 amdgpu_ucode_release(&adev->gfx.mec_fw);
1205                 amdgpu_ucode_release(&adev->gfx.mec2_fw);
1206         }
1207         return err;
1208 }
1209
1210 static void gfx_v8_0_get_csb_buffer(struct amdgpu_device *adev,
1211                                     volatile u32 *buffer)
1212 {
1213         u32 count = 0, i;
1214         const struct cs_section_def *sect = NULL;
1215         const struct cs_extent_def *ext = NULL;
1216
1217         if (adev->gfx.rlc.cs_data == NULL)
1218                 return;
1219         if (buffer == NULL)
1220                 return;
1221
1222         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1223         buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
1224
1225         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
1226         buffer[count++] = cpu_to_le32(0x80000000);
1227         buffer[count++] = cpu_to_le32(0x80000000);
1228
1229         for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) {
1230                 for (ext = sect->section; ext->extent != NULL; ++ext) {
1231                         if (sect->id == SECT_CONTEXT) {
1232                                 buffer[count++] =
1233                                         cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
1234                                 buffer[count++] = cpu_to_le32(ext->reg_index -
1235                                                 PACKET3_SET_CONTEXT_REG_START);
1236                                 for (i = 0; i < ext->reg_count; i++)
1237                                         buffer[count++] = cpu_to_le32(ext->extent[i]);
1238                         } else {
1239                                 return;
1240                         }
1241                 }
1242         }
1243
1244         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2));
1245         buffer[count++] = cpu_to_le32(mmPA_SC_RASTER_CONFIG -
1246                         PACKET3_SET_CONTEXT_REG_START);
1247         buffer[count++] = cpu_to_le32(adev->gfx.config.rb_config[0][0].raster_config);
1248         buffer[count++] = cpu_to_le32(adev->gfx.config.rb_config[0][0].raster_config_1);
1249
1250         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1251         buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
1252
1253         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
1254         buffer[count++] = cpu_to_le32(0);
1255 }
1256
1257 static int gfx_v8_0_cp_jump_table_num(struct amdgpu_device *adev)
1258 {
1259         if (adev->asic_type == CHIP_CARRIZO)
1260                 return 5;
1261         else
1262                 return 4;
1263 }
1264
1265 static int gfx_v8_0_rlc_init(struct amdgpu_device *adev)
1266 {
1267         const struct cs_section_def *cs_data;
1268         int r;
1269
1270         adev->gfx.rlc.cs_data = vi_cs_data;
1271
1272         cs_data = adev->gfx.rlc.cs_data;
1273
1274         if (cs_data) {
1275                 /* init clear state block */
1276                 r = amdgpu_gfx_rlc_init_csb(adev);
1277                 if (r)
1278                         return r;
1279         }
1280
1281         if ((adev->asic_type == CHIP_CARRIZO) ||
1282             (adev->asic_type == CHIP_STONEY)) {
1283                 adev->gfx.rlc.cp_table_size = ALIGN(96 * 5 * 4, 2048) + (64 * 1024); /* JT + GDS */
1284                 r = amdgpu_gfx_rlc_init_cpt(adev);
1285                 if (r)
1286                         return r;
1287         }
1288
1289         /* init spm vmid with 0xf */
1290         if (adev->gfx.rlc.funcs->update_spm_vmid)
1291                 adev->gfx.rlc.funcs->update_spm_vmid(adev, 0xf);
1292
1293         return 0;
1294 }
1295
1296 static void gfx_v8_0_mec_fini(struct amdgpu_device *adev)
1297 {
1298         amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL);
1299 }
1300
1301 static int gfx_v8_0_mec_init(struct amdgpu_device *adev)
1302 {
1303         int r;
1304         u32 *hpd;
1305         size_t mec_hpd_size;
1306
1307         bitmap_zero(adev->gfx.mec.queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
1308
1309         /* take ownership of the relevant compute queues */
1310         amdgpu_gfx_compute_queue_acquire(adev);
1311
1312         mec_hpd_size = adev->gfx.num_compute_rings * GFX8_MEC_HPD_SIZE;
1313         if (mec_hpd_size) {
1314                 r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE,
1315                                               AMDGPU_GEM_DOMAIN_VRAM |
1316                                               AMDGPU_GEM_DOMAIN_GTT,
1317                                               &adev->gfx.mec.hpd_eop_obj,
1318                                               &adev->gfx.mec.hpd_eop_gpu_addr,
1319                                               (void **)&hpd);
1320                 if (r) {
1321                         dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r);
1322                         return r;
1323                 }
1324
1325                 memset(hpd, 0, mec_hpd_size);
1326
1327                 amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj);
1328                 amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj);
1329         }
1330
1331         return 0;
1332 }
1333
1334 static const u32 vgpr_init_compute_shader[] =
1335 {
1336         0x7e000209, 0x7e020208,
1337         0x7e040207, 0x7e060206,
1338         0x7e080205, 0x7e0a0204,
1339         0x7e0c0203, 0x7e0e0202,
1340         0x7e100201, 0x7e120200,
1341         0x7e140209, 0x7e160208,
1342         0x7e180207, 0x7e1a0206,
1343         0x7e1c0205, 0x7e1e0204,
1344         0x7e200203, 0x7e220202,
1345         0x7e240201, 0x7e260200,
1346         0x7e280209, 0x7e2a0208,
1347         0x7e2c0207, 0x7e2e0206,
1348         0x7e300205, 0x7e320204,
1349         0x7e340203, 0x7e360202,
1350         0x7e380201, 0x7e3a0200,
1351         0x7e3c0209, 0x7e3e0208,
1352         0x7e400207, 0x7e420206,
1353         0x7e440205, 0x7e460204,
1354         0x7e480203, 0x7e4a0202,
1355         0x7e4c0201, 0x7e4e0200,
1356         0x7e500209, 0x7e520208,
1357         0x7e540207, 0x7e560206,
1358         0x7e580205, 0x7e5a0204,
1359         0x7e5c0203, 0x7e5e0202,
1360         0x7e600201, 0x7e620200,
1361         0x7e640209, 0x7e660208,
1362         0x7e680207, 0x7e6a0206,
1363         0x7e6c0205, 0x7e6e0204,
1364         0x7e700203, 0x7e720202,
1365         0x7e740201, 0x7e760200,
1366         0x7e780209, 0x7e7a0208,
1367         0x7e7c0207, 0x7e7e0206,
1368         0xbf8a0000, 0xbf810000,
1369 };
1370
1371 static const u32 sgpr_init_compute_shader[] =
1372 {
1373         0xbe8a0100, 0xbe8c0102,
1374         0xbe8e0104, 0xbe900106,
1375         0xbe920108, 0xbe940100,
1376         0xbe960102, 0xbe980104,
1377         0xbe9a0106, 0xbe9c0108,
1378         0xbe9e0100, 0xbea00102,
1379         0xbea20104, 0xbea40106,
1380         0xbea60108, 0xbea80100,
1381         0xbeaa0102, 0xbeac0104,
1382         0xbeae0106, 0xbeb00108,
1383         0xbeb20100, 0xbeb40102,
1384         0xbeb60104, 0xbeb80106,
1385         0xbeba0108, 0xbebc0100,
1386         0xbebe0102, 0xbec00104,
1387         0xbec20106, 0xbec40108,
1388         0xbec60100, 0xbec80102,
1389         0xbee60004, 0xbee70005,
1390         0xbeea0006, 0xbeeb0007,
1391         0xbee80008, 0xbee90009,
1392         0xbefc0000, 0xbf8a0000,
1393         0xbf810000, 0x00000000,
1394 };
1395
1396 static const u32 vgpr_init_regs[] =
1397 {
1398         mmCOMPUTE_STATIC_THREAD_MGMT_SE0, 0xffffffff,
1399         mmCOMPUTE_RESOURCE_LIMITS, 0x1000000, /* CU_GROUP_COUNT=1 */
1400         mmCOMPUTE_NUM_THREAD_X, 256*4,
1401         mmCOMPUTE_NUM_THREAD_Y, 1,
1402         mmCOMPUTE_NUM_THREAD_Z, 1,
1403         mmCOMPUTE_PGM_RSRC1, 0x100004f, /* VGPRS=15 (64 logical VGPRs), SGPRS=1 (16 SGPRs), BULKY=1 */
1404         mmCOMPUTE_PGM_RSRC2, 20,
1405         mmCOMPUTE_USER_DATA_0, 0xedcedc00,
1406         mmCOMPUTE_USER_DATA_1, 0xedcedc01,
1407         mmCOMPUTE_USER_DATA_2, 0xedcedc02,
1408         mmCOMPUTE_USER_DATA_3, 0xedcedc03,
1409         mmCOMPUTE_USER_DATA_4, 0xedcedc04,
1410         mmCOMPUTE_USER_DATA_5, 0xedcedc05,
1411         mmCOMPUTE_USER_DATA_6, 0xedcedc06,
1412         mmCOMPUTE_USER_DATA_7, 0xedcedc07,
1413         mmCOMPUTE_USER_DATA_8, 0xedcedc08,
1414         mmCOMPUTE_USER_DATA_9, 0xedcedc09,
1415 };
1416
1417 static const u32 sgpr1_init_regs[] =
1418 {
1419         mmCOMPUTE_STATIC_THREAD_MGMT_SE0, 0x0f,
1420         mmCOMPUTE_RESOURCE_LIMITS, 0x1000000, /* CU_GROUP_COUNT=1 */
1421         mmCOMPUTE_NUM_THREAD_X, 256*5,
1422         mmCOMPUTE_NUM_THREAD_Y, 1,
1423         mmCOMPUTE_NUM_THREAD_Z, 1,
1424         mmCOMPUTE_PGM_RSRC1, 0x240, /* SGPRS=9 (80 GPRS) */
1425         mmCOMPUTE_PGM_RSRC2, 20,
1426         mmCOMPUTE_USER_DATA_0, 0xedcedc00,
1427         mmCOMPUTE_USER_DATA_1, 0xedcedc01,
1428         mmCOMPUTE_USER_DATA_2, 0xedcedc02,
1429         mmCOMPUTE_USER_DATA_3, 0xedcedc03,
1430         mmCOMPUTE_USER_DATA_4, 0xedcedc04,
1431         mmCOMPUTE_USER_DATA_5, 0xedcedc05,
1432         mmCOMPUTE_USER_DATA_6, 0xedcedc06,
1433         mmCOMPUTE_USER_DATA_7, 0xedcedc07,
1434         mmCOMPUTE_USER_DATA_8, 0xedcedc08,
1435         mmCOMPUTE_USER_DATA_9, 0xedcedc09,
1436 };
1437
1438 static const u32 sgpr2_init_regs[] =
1439 {
1440         mmCOMPUTE_STATIC_THREAD_MGMT_SE0, 0xf0,
1441         mmCOMPUTE_RESOURCE_LIMITS, 0x1000000,
1442         mmCOMPUTE_NUM_THREAD_X, 256*5,
1443         mmCOMPUTE_NUM_THREAD_Y, 1,
1444         mmCOMPUTE_NUM_THREAD_Z, 1,
1445         mmCOMPUTE_PGM_RSRC1, 0x240, /* SGPRS=9 (80 GPRS) */
1446         mmCOMPUTE_PGM_RSRC2, 20,
1447         mmCOMPUTE_USER_DATA_0, 0xedcedc00,
1448         mmCOMPUTE_USER_DATA_1, 0xedcedc01,
1449         mmCOMPUTE_USER_DATA_2, 0xedcedc02,
1450         mmCOMPUTE_USER_DATA_3, 0xedcedc03,
1451         mmCOMPUTE_USER_DATA_4, 0xedcedc04,
1452         mmCOMPUTE_USER_DATA_5, 0xedcedc05,
1453         mmCOMPUTE_USER_DATA_6, 0xedcedc06,
1454         mmCOMPUTE_USER_DATA_7, 0xedcedc07,
1455         mmCOMPUTE_USER_DATA_8, 0xedcedc08,
1456         mmCOMPUTE_USER_DATA_9, 0xedcedc09,
1457 };
1458
1459 static const u32 sec_ded_counter_registers[] =
1460 {
1461         mmCPC_EDC_ATC_CNT,
1462         mmCPC_EDC_SCRATCH_CNT,
1463         mmCPC_EDC_UCODE_CNT,
1464         mmCPF_EDC_ATC_CNT,
1465         mmCPF_EDC_ROQ_CNT,
1466         mmCPF_EDC_TAG_CNT,
1467         mmCPG_EDC_ATC_CNT,
1468         mmCPG_EDC_DMA_CNT,
1469         mmCPG_EDC_TAG_CNT,
1470         mmDC_EDC_CSINVOC_CNT,
1471         mmDC_EDC_RESTORE_CNT,
1472         mmDC_EDC_STATE_CNT,
1473         mmGDS_EDC_CNT,
1474         mmGDS_EDC_GRBM_CNT,
1475         mmGDS_EDC_OA_DED,
1476         mmSPI_EDC_CNT,
1477         mmSQC_ATC_EDC_GATCL1_CNT,
1478         mmSQC_EDC_CNT,
1479         mmSQ_EDC_DED_CNT,
1480         mmSQ_EDC_INFO,
1481         mmSQ_EDC_SEC_CNT,
1482         mmTCC_EDC_CNT,
1483         mmTCP_ATC_EDC_GATCL1_CNT,
1484         mmTCP_EDC_CNT,
1485         mmTD_EDC_CNT
1486 };
1487
1488 static int gfx_v8_0_do_edc_gpr_workarounds(struct amdgpu_device *adev)
1489 {
1490         struct amdgpu_ring *ring = &adev->gfx.compute_ring[0];
1491         struct amdgpu_ib ib;
1492         struct dma_fence *f = NULL;
1493         int r, i;
1494         u32 tmp;
1495         unsigned total_size, vgpr_offset, sgpr_offset;
1496         u64 gpu_addr;
1497
1498         /* only supported on CZ */
1499         if (adev->asic_type != CHIP_CARRIZO)
1500                 return 0;
1501
1502         /* bail if the compute ring is not ready */
1503         if (!ring->sched.ready)
1504                 return 0;
1505
1506         tmp = RREG32(mmGB_EDC_MODE);
1507         WREG32(mmGB_EDC_MODE, 0);
1508
1509         total_size =
1510                 (((ARRAY_SIZE(vgpr_init_regs) / 2) * 3) + 4 + 5 + 2) * 4;
1511         total_size +=
1512                 (((ARRAY_SIZE(sgpr1_init_regs) / 2) * 3) + 4 + 5 + 2) * 4;
1513         total_size +=
1514                 (((ARRAY_SIZE(sgpr2_init_regs) / 2) * 3) + 4 + 5 + 2) * 4;
1515         total_size = ALIGN(total_size, 256);
1516         vgpr_offset = total_size;
1517         total_size += ALIGN(sizeof(vgpr_init_compute_shader), 256);
1518         sgpr_offset = total_size;
1519         total_size += sizeof(sgpr_init_compute_shader);
1520
1521         /* allocate an indirect buffer to put the commands in */
1522         memset(&ib, 0, sizeof(ib));
1523         r = amdgpu_ib_get(adev, NULL, total_size,
1524                                         AMDGPU_IB_POOL_DIRECT, &ib);
1525         if (r) {
1526                 DRM_ERROR("amdgpu: failed to get ib (%d).\n", r);
1527                 return r;
1528         }
1529
1530         /* load the compute shaders */
1531         for (i = 0; i < ARRAY_SIZE(vgpr_init_compute_shader); i++)
1532                 ib.ptr[i + (vgpr_offset / 4)] = vgpr_init_compute_shader[i];
1533
1534         for (i = 0; i < ARRAY_SIZE(sgpr_init_compute_shader); i++)
1535                 ib.ptr[i + (sgpr_offset / 4)] = sgpr_init_compute_shader[i];
1536
1537         /* init the ib length to 0 */
1538         ib.length_dw = 0;
1539
1540         /* VGPR */
1541         /* write the register state for the compute dispatch */
1542         for (i = 0; i < ARRAY_SIZE(vgpr_init_regs); i += 2) {
1543                 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
1544                 ib.ptr[ib.length_dw++] = vgpr_init_regs[i] - PACKET3_SET_SH_REG_START;
1545                 ib.ptr[ib.length_dw++] = vgpr_init_regs[i + 1];
1546         }
1547         /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
1548         gpu_addr = (ib.gpu_addr + (u64)vgpr_offset) >> 8;
1549         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
1550         ib.ptr[ib.length_dw++] = mmCOMPUTE_PGM_LO - PACKET3_SET_SH_REG_START;
1551         ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
1552         ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
1553
1554         /* write dispatch packet */
1555         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
1556         ib.ptr[ib.length_dw++] = 8; /* x */
1557         ib.ptr[ib.length_dw++] = 1; /* y */
1558         ib.ptr[ib.length_dw++] = 1; /* z */
1559         ib.ptr[ib.length_dw++] =
1560                 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
1561
1562         /* write CS partial flush packet */
1563         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
1564         ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
1565
1566         /* SGPR1 */
1567         /* write the register state for the compute dispatch */
1568         for (i = 0; i < ARRAY_SIZE(sgpr1_init_regs); i += 2) {
1569                 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
1570                 ib.ptr[ib.length_dw++] = sgpr1_init_regs[i] - PACKET3_SET_SH_REG_START;
1571                 ib.ptr[ib.length_dw++] = sgpr1_init_regs[i + 1];
1572         }
1573         /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
1574         gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8;
1575         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
1576         ib.ptr[ib.length_dw++] = mmCOMPUTE_PGM_LO - PACKET3_SET_SH_REG_START;
1577         ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
1578         ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
1579
1580         /* write dispatch packet */
1581         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
1582         ib.ptr[ib.length_dw++] = 8; /* x */
1583         ib.ptr[ib.length_dw++] = 1; /* y */
1584         ib.ptr[ib.length_dw++] = 1; /* z */
1585         ib.ptr[ib.length_dw++] =
1586                 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
1587
1588         /* write CS partial flush packet */
1589         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
1590         ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
1591
1592         /* SGPR2 */
1593         /* write the register state for the compute dispatch */
1594         for (i = 0; i < ARRAY_SIZE(sgpr2_init_regs); i += 2) {
1595                 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
1596                 ib.ptr[ib.length_dw++] = sgpr2_init_regs[i] - PACKET3_SET_SH_REG_START;
1597                 ib.ptr[ib.length_dw++] = sgpr2_init_regs[i + 1];
1598         }
1599         /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
1600         gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8;
1601         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
1602         ib.ptr[ib.length_dw++] = mmCOMPUTE_PGM_LO - PACKET3_SET_SH_REG_START;
1603         ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
1604         ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
1605
1606         /* write dispatch packet */
1607         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
1608         ib.ptr[ib.length_dw++] = 8; /* x */
1609         ib.ptr[ib.length_dw++] = 1; /* y */
1610         ib.ptr[ib.length_dw++] = 1; /* z */
1611         ib.ptr[ib.length_dw++] =
1612                 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
1613
1614         /* write CS partial flush packet */
1615         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
1616         ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
1617
1618         /* shedule the ib on the ring */
1619         r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
1620         if (r) {
1621                 DRM_ERROR("amdgpu: ib submit failed (%d).\n", r);
1622                 goto fail;
1623         }
1624
1625         /* wait for the GPU to finish processing the IB */
1626         r = dma_fence_wait(f, false);
1627         if (r) {
1628                 DRM_ERROR("amdgpu: fence wait failed (%d).\n", r);
1629                 goto fail;
1630         }
1631
1632         tmp = REG_SET_FIELD(tmp, GB_EDC_MODE, DED_MODE, 2);
1633         tmp = REG_SET_FIELD(tmp, GB_EDC_MODE, PROP_FED, 1);
1634         WREG32(mmGB_EDC_MODE, tmp);
1635
1636         tmp = RREG32(mmCC_GC_EDC_CONFIG);
1637         tmp = REG_SET_FIELD(tmp, CC_GC_EDC_CONFIG, DIS_EDC, 0) | 1;
1638         WREG32(mmCC_GC_EDC_CONFIG, tmp);
1639
1640
1641         /* read back registers to clear the counters */
1642         for (i = 0; i < ARRAY_SIZE(sec_ded_counter_registers); i++)
1643                 RREG32(sec_ded_counter_registers[i]);
1644
1645 fail:
1646         amdgpu_ib_free(adev, &ib, NULL);
1647         dma_fence_put(f);
1648
1649         return r;
1650 }
1651
1652 static int gfx_v8_0_gpu_early_init(struct amdgpu_device *adev)
1653 {
1654         u32 gb_addr_config;
1655         u32 mc_arb_ramcfg;
1656         u32 dimm00_addr_map, dimm01_addr_map, dimm10_addr_map, dimm11_addr_map;
1657         u32 tmp;
1658         int ret;
1659
1660         switch (adev->asic_type) {
1661         case CHIP_TOPAZ:
1662                 adev->gfx.config.max_shader_engines = 1;
1663                 adev->gfx.config.max_tile_pipes = 2;
1664                 adev->gfx.config.max_cu_per_sh = 6;
1665                 adev->gfx.config.max_sh_per_se = 1;
1666                 adev->gfx.config.max_backends_per_se = 2;
1667                 adev->gfx.config.max_texture_channel_caches = 2;
1668                 adev->gfx.config.max_gprs = 256;
1669                 adev->gfx.config.max_gs_threads = 32;
1670                 adev->gfx.config.max_hw_contexts = 8;
1671
1672                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1673                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1674                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1675                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
1676                 gb_addr_config = TOPAZ_GB_ADDR_CONFIG_GOLDEN;
1677                 break;
1678         case CHIP_FIJI:
1679                 adev->gfx.config.max_shader_engines = 4;
1680                 adev->gfx.config.max_tile_pipes = 16;
1681                 adev->gfx.config.max_cu_per_sh = 16;
1682                 adev->gfx.config.max_sh_per_se = 1;
1683                 adev->gfx.config.max_backends_per_se = 4;
1684                 adev->gfx.config.max_texture_channel_caches = 16;
1685                 adev->gfx.config.max_gprs = 256;
1686                 adev->gfx.config.max_gs_threads = 32;
1687                 adev->gfx.config.max_hw_contexts = 8;
1688
1689                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1690                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1691                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1692                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
1693                 gb_addr_config = TONGA_GB_ADDR_CONFIG_GOLDEN;
1694                 break;
1695         case CHIP_POLARIS11:
1696         case CHIP_POLARIS12:
1697                 ret = amdgpu_atombios_get_gfx_info(adev);
1698                 if (ret)
1699                         return ret;
1700                 adev->gfx.config.max_gprs = 256;
1701                 adev->gfx.config.max_gs_threads = 32;
1702                 adev->gfx.config.max_hw_contexts = 8;
1703
1704                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1705                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1706                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1707                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
1708                 gb_addr_config = POLARIS11_GB_ADDR_CONFIG_GOLDEN;
1709                 break;
1710         case CHIP_POLARIS10:
1711         case CHIP_VEGAM:
1712                 ret = amdgpu_atombios_get_gfx_info(adev);
1713                 if (ret)
1714                         return ret;
1715                 adev->gfx.config.max_gprs = 256;
1716                 adev->gfx.config.max_gs_threads = 32;
1717                 adev->gfx.config.max_hw_contexts = 8;
1718
1719                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1720                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1721                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1722                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
1723                 gb_addr_config = TONGA_GB_ADDR_CONFIG_GOLDEN;
1724                 break;
1725         case CHIP_TONGA:
1726                 adev->gfx.config.max_shader_engines = 4;
1727                 adev->gfx.config.max_tile_pipes = 8;
1728                 adev->gfx.config.max_cu_per_sh = 8;
1729                 adev->gfx.config.max_sh_per_se = 1;
1730                 adev->gfx.config.max_backends_per_se = 2;
1731                 adev->gfx.config.max_texture_channel_caches = 8;
1732                 adev->gfx.config.max_gprs = 256;
1733                 adev->gfx.config.max_gs_threads = 32;
1734                 adev->gfx.config.max_hw_contexts = 8;
1735
1736                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1737                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1738                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1739                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
1740                 gb_addr_config = TONGA_GB_ADDR_CONFIG_GOLDEN;
1741                 break;
1742         case CHIP_CARRIZO:
1743                 adev->gfx.config.max_shader_engines = 1;
1744                 adev->gfx.config.max_tile_pipes = 2;
1745                 adev->gfx.config.max_sh_per_se = 1;
1746                 adev->gfx.config.max_backends_per_se = 2;
1747                 adev->gfx.config.max_cu_per_sh = 8;
1748                 adev->gfx.config.max_texture_channel_caches = 2;
1749                 adev->gfx.config.max_gprs = 256;
1750                 adev->gfx.config.max_gs_threads = 32;
1751                 adev->gfx.config.max_hw_contexts = 8;
1752
1753                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1754                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1755                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1756                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
1757                 gb_addr_config = CARRIZO_GB_ADDR_CONFIG_GOLDEN;
1758                 break;
1759         case CHIP_STONEY:
1760                 adev->gfx.config.max_shader_engines = 1;
1761                 adev->gfx.config.max_tile_pipes = 2;
1762                 adev->gfx.config.max_sh_per_se = 1;
1763                 adev->gfx.config.max_backends_per_se = 1;
1764                 adev->gfx.config.max_cu_per_sh = 3;
1765                 adev->gfx.config.max_texture_channel_caches = 2;
1766                 adev->gfx.config.max_gprs = 256;
1767                 adev->gfx.config.max_gs_threads = 16;
1768                 adev->gfx.config.max_hw_contexts = 8;
1769
1770                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1771                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1772                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1773                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
1774                 gb_addr_config = CARRIZO_GB_ADDR_CONFIG_GOLDEN;
1775                 break;
1776         default:
1777                 adev->gfx.config.max_shader_engines = 2;
1778                 adev->gfx.config.max_tile_pipes = 4;
1779                 adev->gfx.config.max_cu_per_sh = 2;
1780                 adev->gfx.config.max_sh_per_se = 1;
1781                 adev->gfx.config.max_backends_per_se = 2;
1782                 adev->gfx.config.max_texture_channel_caches = 4;
1783                 adev->gfx.config.max_gprs = 256;
1784                 adev->gfx.config.max_gs_threads = 32;
1785                 adev->gfx.config.max_hw_contexts = 8;
1786
1787                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1788                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1789                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1790                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
1791                 gb_addr_config = TONGA_GB_ADDR_CONFIG_GOLDEN;
1792                 break;
1793         }
1794
1795         adev->gfx.config.mc_arb_ramcfg = RREG32(mmMC_ARB_RAMCFG);
1796         mc_arb_ramcfg = adev->gfx.config.mc_arb_ramcfg;
1797
1798         adev->gfx.config.num_banks = REG_GET_FIELD(mc_arb_ramcfg,
1799                                 MC_ARB_RAMCFG, NOOFBANK);
1800         adev->gfx.config.num_ranks = REG_GET_FIELD(mc_arb_ramcfg,
1801                                 MC_ARB_RAMCFG, NOOFRANKS);
1802
1803         adev->gfx.config.num_tile_pipes = adev->gfx.config.max_tile_pipes;
1804         adev->gfx.config.mem_max_burst_length_bytes = 256;
1805         if (adev->flags & AMD_IS_APU) {
1806                 /* Get memory bank mapping mode. */
1807                 tmp = RREG32(mmMC_FUS_DRAM0_BANK_ADDR_MAPPING);
1808                 dimm00_addr_map = REG_GET_FIELD(tmp, MC_FUS_DRAM0_BANK_ADDR_MAPPING, DIMM0ADDRMAP);
1809                 dimm01_addr_map = REG_GET_FIELD(tmp, MC_FUS_DRAM0_BANK_ADDR_MAPPING, DIMM1ADDRMAP);
1810
1811                 tmp = RREG32(mmMC_FUS_DRAM1_BANK_ADDR_MAPPING);
1812                 dimm10_addr_map = REG_GET_FIELD(tmp, MC_FUS_DRAM1_BANK_ADDR_MAPPING, DIMM0ADDRMAP);
1813                 dimm11_addr_map = REG_GET_FIELD(tmp, MC_FUS_DRAM1_BANK_ADDR_MAPPING, DIMM1ADDRMAP);
1814
1815                 /* Validate settings in case only one DIMM installed. */
1816                 if ((dimm00_addr_map == 0) || (dimm00_addr_map == 3) || (dimm00_addr_map == 4) || (dimm00_addr_map > 12))
1817                         dimm00_addr_map = 0;
1818                 if ((dimm01_addr_map == 0) || (dimm01_addr_map == 3) || (dimm01_addr_map == 4) || (dimm01_addr_map > 12))
1819                         dimm01_addr_map = 0;
1820                 if ((dimm10_addr_map == 0) || (dimm10_addr_map == 3) || (dimm10_addr_map == 4) || (dimm10_addr_map > 12))
1821                         dimm10_addr_map = 0;
1822                 if ((dimm11_addr_map == 0) || (dimm11_addr_map == 3) || (dimm11_addr_map == 4) || (dimm11_addr_map > 12))
1823                         dimm11_addr_map = 0;
1824
1825                 /* If DIMM Addr map is 8GB, ROW size should be 2KB. Otherwise 1KB. */
1826                 /* If ROW size(DIMM1) != ROW size(DMIMM0), ROW size should be larger one. */
1827                 if ((dimm00_addr_map == 11) || (dimm01_addr_map == 11) || (dimm10_addr_map == 11) || (dimm11_addr_map == 11))
1828                         adev->gfx.config.mem_row_size_in_kb = 2;
1829                 else
1830                         adev->gfx.config.mem_row_size_in_kb = 1;
1831         } else {
1832                 tmp = REG_GET_FIELD(mc_arb_ramcfg, MC_ARB_RAMCFG, NOOFCOLS);
1833                 adev->gfx.config.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
1834                 if (adev->gfx.config.mem_row_size_in_kb > 4)
1835                         adev->gfx.config.mem_row_size_in_kb = 4;
1836         }
1837
1838         adev->gfx.config.shader_engine_tile_size = 32;
1839         adev->gfx.config.num_gpus = 1;
1840         adev->gfx.config.multi_gpu_tile_size = 64;
1841
1842         /* fix up row size */
1843         switch (adev->gfx.config.mem_row_size_in_kb) {
1844         case 1:
1845         default:
1846                 gb_addr_config = REG_SET_FIELD(gb_addr_config, GB_ADDR_CONFIG, ROW_SIZE, 0);
1847                 break;
1848         case 2:
1849                 gb_addr_config = REG_SET_FIELD(gb_addr_config, GB_ADDR_CONFIG, ROW_SIZE, 1);
1850                 break;
1851         case 4:
1852                 gb_addr_config = REG_SET_FIELD(gb_addr_config, GB_ADDR_CONFIG, ROW_SIZE, 2);
1853                 break;
1854         }
1855         adev->gfx.config.gb_addr_config = gb_addr_config;
1856
1857         return 0;
1858 }
1859
1860 static int gfx_v8_0_compute_ring_init(struct amdgpu_device *adev, int ring_id,
1861                                         int mec, int pipe, int queue)
1862 {
1863         int r;
1864         unsigned irq_type;
1865         struct amdgpu_ring *ring = &adev->gfx.compute_ring[ring_id];
1866         unsigned int hw_prio;
1867
1868         ring = &adev->gfx.compute_ring[ring_id];
1869
1870         /* mec0 is me1 */
1871         ring->me = mec + 1;
1872         ring->pipe = pipe;
1873         ring->queue = queue;
1874
1875         ring->ring_obj = NULL;
1876         ring->use_doorbell = true;
1877         ring->doorbell_index = adev->doorbell_index.mec_ring0 + ring_id;
1878         ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr
1879                                 + (ring_id * GFX8_MEC_HPD_SIZE);
1880         sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue);
1881
1882         irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
1883                 + ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec)
1884                 + ring->pipe;
1885
1886         hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring) ?
1887                         AMDGPU_RING_PRIO_2 : AMDGPU_RING_PRIO_DEFAULT;
1888         /* type-2 packets are deprecated on MEC, use type-3 instead */
1889         r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type,
1890                              hw_prio, NULL);
1891         if (r)
1892                 return r;
1893
1894
1895         return 0;
1896 }
1897
1898 static void gfx_v8_0_sq_irq_work_func(struct work_struct *work);
1899
1900 static int gfx_v8_0_sw_init(void *handle)
1901 {
1902         int i, j, k, r, ring_id;
1903         struct amdgpu_ring *ring;
1904         struct amdgpu_kiq *kiq;
1905         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1906
1907         switch (adev->asic_type) {
1908         case CHIP_TONGA:
1909         case CHIP_CARRIZO:
1910         case CHIP_FIJI:
1911         case CHIP_POLARIS10:
1912         case CHIP_POLARIS11:
1913         case CHIP_POLARIS12:
1914         case CHIP_VEGAM:
1915                 adev->gfx.mec.num_mec = 2;
1916                 break;
1917         case CHIP_TOPAZ:
1918         case CHIP_STONEY:
1919         default:
1920                 adev->gfx.mec.num_mec = 1;
1921                 break;
1922         }
1923
1924         adev->gfx.mec.num_pipe_per_mec = 4;
1925         adev->gfx.mec.num_queue_per_pipe = 8;
1926
1927         /* EOP Event */
1928         r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_CP_END_OF_PIPE, &adev->gfx.eop_irq);
1929         if (r)
1930                 return r;
1931
1932         /* Privileged reg */
1933         r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_CP_PRIV_REG_FAULT,
1934                               &adev->gfx.priv_reg_irq);
1935         if (r)
1936                 return r;
1937
1938         /* Privileged inst */
1939         r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_CP_PRIV_INSTR_FAULT,
1940                               &adev->gfx.priv_inst_irq);
1941         if (r)
1942                 return r;
1943
1944         /* Add CP EDC/ECC irq  */
1945         r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_CP_ECC_ERROR,
1946                               &adev->gfx.cp_ecc_error_irq);
1947         if (r)
1948                 return r;
1949
1950         /* SQ interrupts. */
1951         r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_SQ_INTERRUPT_MSG,
1952                               &adev->gfx.sq_irq);
1953         if (r) {
1954                 DRM_ERROR("amdgpu_irq_add() for SQ failed: %d\n", r);
1955                 return r;
1956         }
1957
1958         INIT_WORK(&adev->gfx.sq_work.work, gfx_v8_0_sq_irq_work_func);
1959
1960         adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE;
1961
1962         r = gfx_v8_0_init_microcode(adev);
1963         if (r) {
1964                 DRM_ERROR("Failed to load gfx firmware!\n");
1965                 return r;
1966         }
1967
1968         r = adev->gfx.rlc.funcs->init(adev);
1969         if (r) {
1970                 DRM_ERROR("Failed to init rlc BOs!\n");
1971                 return r;
1972         }
1973
1974         r = gfx_v8_0_mec_init(adev);
1975         if (r) {
1976                 DRM_ERROR("Failed to init MEC BOs!\n");
1977                 return r;
1978         }
1979
1980         /* set up the gfx ring */
1981         for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
1982                 ring = &adev->gfx.gfx_ring[i];
1983                 ring->ring_obj = NULL;
1984                 sprintf(ring->name, "gfx");
1985                 /* no gfx doorbells on iceland */
1986                 if (adev->asic_type != CHIP_TOPAZ) {
1987                         ring->use_doorbell = true;
1988                         ring->doorbell_index = adev->doorbell_index.gfx_ring0;
1989                 }
1990
1991                 r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq,
1992                                      AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP,
1993                                      AMDGPU_RING_PRIO_DEFAULT, NULL);
1994                 if (r)
1995                         return r;
1996         }
1997
1998
1999         /* set up the compute queues - allocate horizontally across pipes */
2000         ring_id = 0;
2001         for (i = 0; i < adev->gfx.mec.num_mec; ++i) {
2002                 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {
2003                         for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {
2004                                 if (!amdgpu_gfx_is_mec_queue_enabled(adev, i, k, j))
2005                                         continue;
2006
2007                                 r = gfx_v8_0_compute_ring_init(adev,
2008                                                                 ring_id,
2009                                                                 i, k, j);
2010                                 if (r)
2011                                         return r;
2012
2013                                 ring_id++;
2014                         }
2015                 }
2016         }
2017
2018         r = amdgpu_gfx_kiq_init(adev, GFX8_MEC_HPD_SIZE);
2019         if (r) {
2020                 DRM_ERROR("Failed to init KIQ BOs!\n");
2021                 return r;
2022         }
2023
2024         kiq = &adev->gfx.kiq;
2025         r = amdgpu_gfx_kiq_init_ring(adev, &kiq->ring, &kiq->irq);
2026         if (r)
2027                 return r;
2028
2029         /* create MQD for all compute queues as well as KIQ for SRIOV case */
2030         r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct vi_mqd_allocation));
2031         if (r)
2032                 return r;
2033
2034         adev->gfx.ce_ram_size = 0x8000;
2035
2036         r = gfx_v8_0_gpu_early_init(adev);
2037         if (r)
2038                 return r;
2039
2040         return 0;
2041 }
2042
2043 static int gfx_v8_0_sw_fini(void *handle)
2044 {
2045         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2046         int i;
2047
2048         for (i = 0; i < adev->gfx.num_gfx_rings; i++)
2049                 amdgpu_ring_fini(&adev->gfx.gfx_ring[i]);
2050         for (i = 0; i < adev->gfx.num_compute_rings; i++)
2051                 amdgpu_ring_fini(&adev->gfx.compute_ring[i]);
2052
2053         amdgpu_gfx_mqd_sw_fini(adev);
2054         amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq.ring);
2055         amdgpu_gfx_kiq_fini(adev);
2056
2057         gfx_v8_0_mec_fini(adev);
2058         amdgpu_gfx_rlc_fini(adev);
2059         amdgpu_bo_free_kernel(&adev->gfx.rlc.clear_state_obj,
2060                                 &adev->gfx.rlc.clear_state_gpu_addr,
2061                                 (void **)&adev->gfx.rlc.cs_ptr);
2062         if ((adev->asic_type == CHIP_CARRIZO) ||
2063             (adev->asic_type == CHIP_STONEY)) {
2064                 amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj,
2065                                 &adev->gfx.rlc.cp_table_gpu_addr,
2066                                 (void **)&adev->gfx.rlc.cp_table_ptr);
2067         }
2068         gfx_v8_0_free_microcode(adev);
2069
2070         return 0;
2071 }
2072
2073 static void gfx_v8_0_tiling_mode_table_init(struct amdgpu_device *adev)
2074 {
2075         uint32_t *modearray, *mod2array;
2076         const u32 num_tile_mode_states = ARRAY_SIZE(adev->gfx.config.tile_mode_array);
2077         const u32 num_secondary_tile_mode_states = ARRAY_SIZE(adev->gfx.config.macrotile_mode_array);
2078         u32 reg_offset;
2079
2080         modearray = adev->gfx.config.tile_mode_array;
2081         mod2array = adev->gfx.config.macrotile_mode_array;
2082
2083         for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2084                 modearray[reg_offset] = 0;
2085
2086         for (reg_offset = 0; reg_offset <  num_secondary_tile_mode_states; reg_offset++)
2087                 mod2array[reg_offset] = 0;
2088
2089         switch (adev->asic_type) {
2090         case CHIP_TOPAZ:
2091                 modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2092                                 PIPE_CONFIG(ADDR_SURF_P2) |
2093                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2094                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2095                 modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2096                                 PIPE_CONFIG(ADDR_SURF_P2) |
2097                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2098                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2099                 modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2100                                 PIPE_CONFIG(ADDR_SURF_P2) |
2101                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2102                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2103                 modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2104                                 PIPE_CONFIG(ADDR_SURF_P2) |
2105                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2106                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2107                 modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2108                                 PIPE_CONFIG(ADDR_SURF_P2) |
2109                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2110                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2111                 modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2112                                 PIPE_CONFIG(ADDR_SURF_P2) |
2113                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2114                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2115                 modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2116                                 PIPE_CONFIG(ADDR_SURF_P2) |
2117                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2118                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2119                 modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2120                                 PIPE_CONFIG(ADDR_SURF_P2));
2121                 modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2122                                 PIPE_CONFIG(ADDR_SURF_P2) |
2123                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2124                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2125                 modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2126                                  PIPE_CONFIG(ADDR_SURF_P2) |
2127                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2128                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2129                 modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2130                                  PIPE_CONFIG(ADDR_SURF_P2) |
2131                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2132                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2133                 modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2134                                  PIPE_CONFIG(ADDR_SURF_P2) |
2135                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2136                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2137                 modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2138                                  PIPE_CONFIG(ADDR_SURF_P2) |
2139                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2140                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2141                 modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) |
2142                                  PIPE_CONFIG(ADDR_SURF_P2) |
2143                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2144                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2145                 modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2146                                  PIPE_CONFIG(ADDR_SURF_P2) |
2147                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2148                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2149                 modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
2150                                  PIPE_CONFIG(ADDR_SURF_P2) |
2151                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2152                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2153                 modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
2154                                  PIPE_CONFIG(ADDR_SURF_P2) |
2155                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2156                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2157                 modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
2158                                  PIPE_CONFIG(ADDR_SURF_P2) |
2159                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2160                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2161                 modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) |
2162                                  PIPE_CONFIG(ADDR_SURF_P2) |
2163                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2164                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2165                 modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
2166                                  PIPE_CONFIG(ADDR_SURF_P2) |
2167                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2168                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2169                 modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
2170                                  PIPE_CONFIG(ADDR_SURF_P2) |
2171                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2172                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2173                 modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |
2174                                  PIPE_CONFIG(ADDR_SURF_P2) |
2175                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2176                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2177                 modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) |
2178                                  PIPE_CONFIG(ADDR_SURF_P2) |
2179                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2180                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2181                 modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2182                                  PIPE_CONFIG(ADDR_SURF_P2) |
2183                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2184                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2185                 modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2186                                  PIPE_CONFIG(ADDR_SURF_P2) |
2187                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2188                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2189                 modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2190                                  PIPE_CONFIG(ADDR_SURF_P2) |
2191                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2192                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2193
2194                 mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2195                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2196                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2197                                 NUM_BANKS(ADDR_SURF_8_BANK));
2198                 mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2199                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2200                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2201                                 NUM_BANKS(ADDR_SURF_8_BANK));
2202                 mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2203                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2204                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2205                                 NUM_BANKS(ADDR_SURF_8_BANK));
2206                 mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2207                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2208                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2209                                 NUM_BANKS(ADDR_SURF_8_BANK));
2210                 mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2211                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2212                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2213                                 NUM_BANKS(ADDR_SURF_8_BANK));
2214                 mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2215                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2216                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2217                                 NUM_BANKS(ADDR_SURF_8_BANK));
2218                 mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2219                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2220                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2221                                 NUM_BANKS(ADDR_SURF_8_BANK));
2222                 mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2223                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2224                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2225                                 NUM_BANKS(ADDR_SURF_16_BANK));
2226                 mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2227                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2228                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2229                                 NUM_BANKS(ADDR_SURF_16_BANK));
2230                 mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2231                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2232                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2233                                  NUM_BANKS(ADDR_SURF_16_BANK));
2234                 mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2235                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2236                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2237                                  NUM_BANKS(ADDR_SURF_16_BANK));
2238                 mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2239                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2240                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2241                                  NUM_BANKS(ADDR_SURF_16_BANK));
2242                 mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2243                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2244                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2245                                  NUM_BANKS(ADDR_SURF_16_BANK));
2246                 mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2247                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2248                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2249                                  NUM_BANKS(ADDR_SURF_8_BANK));
2250
2251                 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2252                         if (reg_offset != 7 && reg_offset != 12 && reg_offset != 17 &&
2253                             reg_offset != 23)
2254                                 WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]);
2255
2256                 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
2257                         if (reg_offset != 7)
2258                                 WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]);
2259
2260                 break;
2261         case CHIP_FIJI:
2262         case CHIP_VEGAM:
2263                 modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2264                                 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2265                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2266                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2267                 modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2268                                 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2269                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2270                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2271                 modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2272                                 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2273                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2274                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2275                 modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2276                                 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2277                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2278                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2279                 modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2280                                 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2281                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2282                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2283                 modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2284                                 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2285                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2286                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2287                 modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2288                                 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2289                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2290                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2291                 modearray[7] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2292                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2293                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2294                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2295                 modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2296                                 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16));
2297                 modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2298                                 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2299                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2300                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2301                 modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2302                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2303                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2304                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2305                 modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2306                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2307                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2308                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2309                 modearray[12] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2310                                  PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2311                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2312                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2313                 modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2314                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2315                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2316                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2317                 modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2318                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2319                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2320                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2321                 modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) |
2322                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2323                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2324                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2325                 modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2326                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2327                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2328                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2329                 modearray[17] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2330                                  PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2331                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2332                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2333                 modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
2334                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2335                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2336                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2337                 modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
2338                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2339                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2340                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2341                 modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
2342                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2343                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2344                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2345                 modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) |
2346                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2347                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2348                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2349                 modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
2350                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2351                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2352                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2353                 modearray[23] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
2354                                  PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2355                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2356                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2357                 modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
2358                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2359                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2360                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2361                 modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |
2362                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2363                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2364                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2365                 modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) |
2366                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2367                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2368                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2369                 modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2370                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2371                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2372                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2373                 modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2374                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2375                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2376                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2377                 modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2378                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2379                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2380                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2381                 modearray[30] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2382                                  PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2383                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2384                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2385
2386                 mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2387                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2388                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2389                                 NUM_BANKS(ADDR_SURF_8_BANK));
2390                 mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2391                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2392                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2393                                 NUM_BANKS(ADDR_SURF_8_BANK));
2394                 mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2395                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2396                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2397                                 NUM_BANKS(ADDR_SURF_8_BANK));
2398                 mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2399                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2400                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2401                                 NUM_BANKS(ADDR_SURF_8_BANK));
2402                 mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2403                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2404                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2405                                 NUM_BANKS(ADDR_SURF_8_BANK));
2406                 mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2407                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2408                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2409                                 NUM_BANKS(ADDR_SURF_8_BANK));
2410                 mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2411                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2412                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2413                                 NUM_BANKS(ADDR_SURF_8_BANK));
2414                 mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2415                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2416                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2417                                 NUM_BANKS(ADDR_SURF_8_BANK));
2418                 mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2419                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2420                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2421                                 NUM_BANKS(ADDR_SURF_8_BANK));
2422                 mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2423                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2424                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2425                                  NUM_BANKS(ADDR_SURF_8_BANK));
2426                 mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2427                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2428                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2429                                  NUM_BANKS(ADDR_SURF_8_BANK));
2430                 mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2431                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2432                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2433                                  NUM_BANKS(ADDR_SURF_8_BANK));
2434                 mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2435                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2436                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2437                                  NUM_BANKS(ADDR_SURF_8_BANK));
2438                 mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2439                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2440                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2441                                  NUM_BANKS(ADDR_SURF_4_BANK));
2442
2443                 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2444                         WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]);
2445
2446                 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
2447                         if (reg_offset != 7)
2448                                 WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]);
2449
2450                 break;
2451         case CHIP_TONGA:
2452                 modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2453                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2454                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2455                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2456                 modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2457                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2458                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2459                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2460                 modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2461                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2462                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2463                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2464                 modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2465                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2466                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2467                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2468                 modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2469                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2470                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2471                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2472                 modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2473                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2474                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2475                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2476                 modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2477                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2478                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2479                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2480                 modearray[7] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2481                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2482                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2483                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2484                 modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2485                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
2486                 modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2487                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2488                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2489                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2490                 modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2491                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2492                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2493                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2494                 modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2495                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2496                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2497                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2498                 modearray[12] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2499                                  PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2500                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2501                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2502                 modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2503                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2504                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2505                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2506                 modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2507                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2508                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2509                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2510                 modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) |
2511                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2512                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2513                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2514                 modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2515                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2516                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2517                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2518                 modearray[17] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2519                                  PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2520                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2521                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2522                 modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
2523                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2524                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2525                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2526                 modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
2527                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2528                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2529                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2530                 modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
2531                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2532                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2533                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2534                 modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) |
2535                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2536                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2537                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2538                 modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
2539                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2540                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2541                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2542                 modearray[23] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
2543                                  PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2544                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2545                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2546                 modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
2547                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2548                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2549                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2550                 modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |
2551                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2552                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2553                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2554                 modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) |
2555                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2556                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2557                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2558                 modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2559                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2560                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2561                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2562                 modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2563                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2564                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2565                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2566                 modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2567                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2568                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2569                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2570                 modearray[30] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2571                                  PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2572                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2573                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2574
2575                 mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2576                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2577                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2578                                 NUM_BANKS(ADDR_SURF_16_BANK));
2579                 mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2580                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2581                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2582                                 NUM_BANKS(ADDR_SURF_16_BANK));
2583                 mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2584                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2585                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2586                                 NUM_BANKS(ADDR_SURF_16_BANK));
2587                 mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2588                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2589                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2590                                 NUM_BANKS(ADDR_SURF_16_BANK));
2591                 mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2592                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2593                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2594                                 NUM_BANKS(ADDR_SURF_16_BANK));
2595                 mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2596                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2597                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2598                                 NUM_BANKS(ADDR_SURF_16_BANK));
2599                 mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2600                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2601                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2602                                 NUM_BANKS(ADDR_SURF_16_BANK));
2603                 mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2604                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2605                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2606                                 NUM_BANKS(ADDR_SURF_16_BANK));
2607                 mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2608                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2609                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2610                                 NUM_BANKS(ADDR_SURF_16_BANK));
2611                 mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2612                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2613                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2614                                  NUM_BANKS(ADDR_SURF_16_BANK));
2615                 mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2616                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2617                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2618                                  NUM_BANKS(ADDR_SURF_16_BANK));
2619                 mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2620                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2621                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2622                                  NUM_BANKS(ADDR_SURF_8_BANK));
2623                 mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2624                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2625                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2626                                  NUM_BANKS(ADDR_SURF_4_BANK));
2627                 mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2628                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2629                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2630                                  NUM_BANKS(ADDR_SURF_4_BANK));
2631
2632                 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2633                         WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]);
2634
2635                 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
2636                         if (reg_offset != 7)
2637                                 WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]);
2638
2639                 break;
2640         case CHIP_POLARIS11:
2641         case CHIP_POLARIS12:
2642                 modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2643                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2644                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2645                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2646                 modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2647                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2648                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2649                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2650                 modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2651                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2652                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2653                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2654                 modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2655                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2656                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2657                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2658                 modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2659                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2660                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2661                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2662                 modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2663                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2664                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2665                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2666                 modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2667                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2668                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2669                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2670                 modearray[7] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2671                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2672                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2673                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2674                 modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2675                                 PIPE_CONFIG(ADDR_SURF_P4_16x16));
2676                 modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2677                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2678                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2679                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2680                 modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2681                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2682                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2683                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2684                 modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2685                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2686                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2687                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2688                 modearray[12] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2689                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2690                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2691                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2692                 modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2693                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2694                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2695                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2696                 modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2697                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2698                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2699                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2700                 modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) |
2701                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2702                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2703                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2704                 modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2705                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2706                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2707                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2708                 modearray[17] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2709                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2710                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2711                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2712                 modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
2713                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2714                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2715                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2716                 modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
2717                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2718                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2719                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2720                 modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
2721                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2722                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2723                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2724                 modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) |
2725                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2726                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2727                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2728                 modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
2729                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2730                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2731                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2732                 modearray[23] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
2733                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2734                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2735                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2736                 modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
2737                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2738                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2739                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2740                 modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |
2741                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2742                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2743                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2744                 modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) |
2745                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2746                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2747                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2748                 modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2749                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2750                                 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2751                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2752                 modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2753                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2754                                 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2755                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2756                 modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2757                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2758                                 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2759                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2760                 modearray[30] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2761                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2762                                 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2763                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2764
2765                 mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2766                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2767                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2768                                 NUM_BANKS(ADDR_SURF_16_BANK));
2769
2770                 mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2771                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2772                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2773                                 NUM_BANKS(ADDR_SURF_16_BANK));
2774
2775                 mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2776                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2777                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2778                                 NUM_BANKS(ADDR_SURF_16_BANK));
2779
2780                 mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2781                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2782                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2783                                 NUM_BANKS(ADDR_SURF_16_BANK));
2784
2785                 mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2786                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2787                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2788                                 NUM_BANKS(ADDR_SURF_16_BANK));
2789
2790                 mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2791                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2792                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2793                                 NUM_BANKS(ADDR_SURF_16_BANK));
2794
2795                 mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2796                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2797                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2798                                 NUM_BANKS(ADDR_SURF_16_BANK));
2799
2800                 mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2801                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2802                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2803                                 NUM_BANKS(ADDR_SURF_16_BANK));
2804
2805                 mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2806                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2807                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2808                                 NUM_BANKS(ADDR_SURF_16_BANK));
2809
2810                 mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2811                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2812                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2813                                 NUM_BANKS(ADDR_SURF_16_BANK));
2814
2815                 mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2816                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2817                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2818                                 NUM_BANKS(ADDR_SURF_16_BANK));
2819
2820                 mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2821                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2822                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2823                                 NUM_BANKS(ADDR_SURF_16_BANK));
2824
2825                 mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2826                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2827                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2828                                 NUM_BANKS(ADDR_SURF_8_BANK));
2829
2830                 mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2831                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2832                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2833                                 NUM_BANKS(ADDR_SURF_4_BANK));
2834
2835                 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2836                         WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]);
2837
2838                 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
2839                         if (reg_offset != 7)
2840                                 WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]);
2841
2842                 break;
2843         case CHIP_POLARIS10:
2844                 modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2845                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2846                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2847                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2848                 modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2849                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2850                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2851                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2852                 modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2853                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2854                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2855                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2856                 modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2857                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2858                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2859                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2860                 modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2861                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2862                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2863                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2864                 modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2865                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2866                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2867                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2868                 modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2869                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2870                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2871                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2872                 modearray[7] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2873                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2874                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2875                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2876                 modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2877                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
2878                 modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2879                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2880                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2881                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2882                 modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2883                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2884                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2885                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2886                 modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2887                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2888                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2889                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2890                 modearray[12] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2891                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2892                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2893                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2894                 modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2895                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2896                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2897                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2898                 modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2899                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2900                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2901                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2902                 modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) |
2903                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2904                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2905                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2906                 modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2907                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2908                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2909                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2910                 modearray[17] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2911                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2912                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2913                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2914                 modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
2915                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2916                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2917                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2918                 modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
2919                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2920                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2921                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2922                 modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
2923                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2924                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2925                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2926                 modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) |
2927                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2928                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2929                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2930                 modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
2931                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2932                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2933                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2934                 modearray[23] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
2935                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2936                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2937                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2938                 modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
2939                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2940                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2941                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2942                 modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |
2943                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2944                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2945                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2946                 modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) |
2947                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2948                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2949                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2950                 modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2951                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2952                                 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2953                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2954                 modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2955                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2956                                 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2957                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2958                 modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2959                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2960                                 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2961                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2962                 modearray[30] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2963                                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2964                                 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2965                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2966
2967                 mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2968                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2969                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2970                                 NUM_BANKS(ADDR_SURF_16_BANK));
2971
2972                 mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2973                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2974                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2975                                 NUM_BANKS(ADDR_SURF_16_BANK));
2976
2977                 mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2978                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2979                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2980                                 NUM_BANKS(ADDR_SURF_16_BANK));
2981
2982                 mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2983                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2984                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2985                                 NUM_BANKS(ADDR_SURF_16_BANK));
2986
2987                 mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2988                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2989                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2990                                 NUM_BANKS(ADDR_SURF_16_BANK));
2991
2992                 mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2993                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2994                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2995                                 NUM_BANKS(ADDR_SURF_16_BANK));
2996
2997                 mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2998                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2999                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
3000                                 NUM_BANKS(ADDR_SURF_16_BANK));
3001
3002                 mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3003                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
3004                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3005                                 NUM_BANKS(ADDR_SURF_16_BANK));
3006
3007                 mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3008                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3009                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3010                                 NUM_BANKS(ADDR_SURF_16_BANK));
3011
3012                 mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3013                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3014                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3015                                 NUM_BANKS(ADDR_SURF_16_BANK));
3016
3017                 mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3018                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3019                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3020                                 NUM_BANKS(ADDR_SURF_16_BANK));
3021
3022                 mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3023                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3024                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
3025                                 NUM_BANKS(ADDR_SURF_8_BANK));
3026
3027                 mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3028                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3029                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
3030                                 NUM_BANKS(ADDR_SURF_4_BANK));
3031
3032                 mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3033                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3034                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
3035                                 NUM_BANKS(ADDR_SURF_4_BANK));
3036
3037                 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
3038                         WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]);
3039
3040                 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
3041                         if (reg_offset != 7)
3042                                 WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]);
3043
3044                 break;
3045         case CHIP_STONEY:
3046                 modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3047                                 PIPE_CONFIG(ADDR_SURF_P2) |
3048                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
3049                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3050                 modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3051                                 PIPE_CONFIG(ADDR_SURF_P2) |
3052                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
3053                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3054                 modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3055                                 PIPE_CONFIG(ADDR_SURF_P2) |
3056                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
3057                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3058                 modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3059                                 PIPE_CONFIG(ADDR_SURF_P2) |
3060                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
3061                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3062                 modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3063                                 PIPE_CONFIG(ADDR_SURF_P2) |
3064                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
3065                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3066                 modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3067                                 PIPE_CONFIG(ADDR_SURF_P2) |
3068                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
3069                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3070                 modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3071                                 PIPE_CONFIG(ADDR_SURF_P2) |
3072                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
3073                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3074                 modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
3075                                 PIPE_CONFIG(ADDR_SURF_P2));
3076                 modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3077                                 PIPE_CONFIG(ADDR_SURF_P2) |
3078                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3079                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3080                 modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3081                                  PIPE_CONFIG(ADDR_SURF_P2) |
3082                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3083                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3084                 modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3085                                  PIPE_CONFIG(ADDR_SURF_P2) |
3086                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3087                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
3088                 modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3089                                  PIPE_CONFIG(ADDR_SURF_P2) |
3090                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3091                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3092                 modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3093                                  PIPE_CONFIG(ADDR_SURF_P2) |
3094                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3095                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3096                 modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) |
3097                                  PIPE_CONFIG(ADDR_SURF_P2) |
3098                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3099                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3100                 modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3101                                  PIPE_CONFIG(ADDR_SURF_P2) |
3102                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3103                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
3104                 modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
3105                                  PIPE_CONFIG(ADDR_SURF_P2) |
3106                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3107                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3108                 modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
3109                                  PIPE_CONFIG(ADDR_SURF_P2) |
3110                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3111                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3112                 modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
3113                                  PIPE_CONFIG(ADDR_SURF_P2) |
3114                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3115                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3116                 modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) |
3117                                  PIPE_CONFIG(ADDR_SURF_P2) |
3118                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3119                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3120                 modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
3121                                  PIPE_CONFIG(ADDR_SURF_P2) |
3122                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3123                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3124                 modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
3125                                  PIPE_CONFIG(ADDR_SURF_P2) |
3126                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3127                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3128                 modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |
3129                                  PIPE_CONFIG(ADDR_SURF_P2) |
3130                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3131                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3132                 modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) |
3133                                  PIPE_CONFIG(ADDR_SURF_P2) |
3134                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3135                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3136                 modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3137                                  PIPE_CONFIG(ADDR_SURF_P2) |
3138                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3139                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3140                 modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3141                                  PIPE_CONFIG(ADDR_SURF_P2) |
3142                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3143                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3144                 modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3145                                  PIPE_CONFIG(ADDR_SURF_P2) |
3146                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3147                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
3148
3149                 mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3150                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3151                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3152                                 NUM_BANKS(ADDR_SURF_8_BANK));
3153                 mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3154                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3155                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3156                                 NUM_BANKS(ADDR_SURF_8_BANK));
3157                 mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3158                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3159                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3160                                 NUM_BANKS(ADDR_SURF_8_BANK));
3161                 mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3162                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3163                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3164                                 NUM_BANKS(ADDR_SURF_8_BANK));
3165                 mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3166                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3167                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3168                                 NUM_BANKS(ADDR_SURF_8_BANK));
3169                 mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3170                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3171                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3172                                 NUM_BANKS(ADDR_SURF_8_BANK));
3173                 mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3174                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3175                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3176                                 NUM_BANKS(ADDR_SURF_8_BANK));
3177                 mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3178                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
3179                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3180                                 NUM_BANKS(ADDR_SURF_16_BANK));
3181                 mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3182                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3183                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3184                                 NUM_BANKS(ADDR_SURF_16_BANK));
3185                 mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3186                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3187                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3188                                  NUM_BANKS(ADDR_SURF_16_BANK));
3189                 mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3190                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3191                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3192                                  NUM_BANKS(ADDR_SURF_16_BANK));
3193                 mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3194                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3195                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3196                                  NUM_BANKS(ADDR_SURF_16_BANK));
3197                 mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3198                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3199                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3200                                  NUM_BANKS(ADDR_SURF_16_BANK));
3201                 mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3202                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3203                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3204                                  NUM_BANKS(ADDR_SURF_8_BANK));
3205
3206                 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
3207                         if (reg_offset != 7 && reg_offset != 12 && reg_offset != 17 &&
3208                             reg_offset != 23)
3209                                 WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]);
3210
3211                 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
3212                         if (reg_offset != 7)
3213                                 WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]);
3214
3215                 break;
3216         default:
3217                 dev_warn(adev->dev,
3218                          "Unknown chip type (%d) in function gfx_v8_0_tiling_mode_table_init() falling through to CHIP_CARRIZO\n",
3219                          adev->asic_type);
3220                 fallthrough;
3221
3222         case CHIP_CARRIZO:
3223                 modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3224                                 PIPE_CONFIG(ADDR_SURF_P2) |
3225                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
3226                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3227                 modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3228                                 PIPE_CONFIG(ADDR_SURF_P2) |
3229                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
3230                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3231                 modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3232                                 PIPE_CONFIG(ADDR_SURF_P2) |
3233                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
3234                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3235                 modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3236                                 PIPE_CONFIG(ADDR_SURF_P2) |
3237                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
3238                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3239                 modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3240                                 PIPE_CONFIG(ADDR_SURF_P2) |
3241                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
3242                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3243                 modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3244                                 PIPE_CONFIG(ADDR_SURF_P2) |
3245                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
3246                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3247                 modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3248                                 PIPE_CONFIG(ADDR_SURF_P2) |
3249                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
3250                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3251                 modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
3252                                 PIPE_CONFIG(ADDR_SURF_P2));
3253                 modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3254                                 PIPE_CONFIG(ADDR_SURF_P2) |
3255                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3256                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3257                 modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3258                                  PIPE_CONFIG(ADDR_SURF_P2) |
3259                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3260                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3261                 modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3262                                  PIPE_CONFIG(ADDR_SURF_P2) |
3263                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3264                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
3265                 modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3266                                  PIPE_CONFIG(ADDR_SURF_P2) |
3267                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3268                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3269                 modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3270                                  PIPE_CONFIG(ADDR_SURF_P2) |
3271                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3272                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3273                 modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) |
3274                                  PIPE_CONFIG(ADDR_SURF_P2) |
3275                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3276                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3277                 modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3278                                  PIPE_CONFIG(ADDR_SURF_P2) |
3279                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3280                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
3281                 modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
3282                                  PIPE_CONFIG(ADDR_SURF_P2) |
3283                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3284                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3285                 modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
3286                                  PIPE_CONFIG(ADDR_SURF_P2) |
3287                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3288                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3289                 modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
3290                                  PIPE_CONFIG(ADDR_SURF_P2) |
3291                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3292                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3293                 modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) |
3294                                  PIPE_CONFIG(ADDR_SURF_P2) |
3295                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3296                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3297                 modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
3298                                  PIPE_CONFIG(ADDR_SURF_P2) |
3299                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3300                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3301                 modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
3302                                  PIPE_CONFIG(ADDR_SURF_P2) |
3303                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3304                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3305                 modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |
3306                                  PIPE_CONFIG(ADDR_SURF_P2) |
3307                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3308                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3309                 modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) |
3310                                  PIPE_CONFIG(ADDR_SURF_P2) |
3311                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3312                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3313                 modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3314                                  PIPE_CONFIG(ADDR_SURF_P2) |
3315                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3316                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3317                 modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3318                                  PIPE_CONFIG(ADDR_SURF_P2) |
3319                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3320                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3321                 modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3322                                  PIPE_CONFIG(ADDR_SURF_P2) |
3323                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3324                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
3325
3326                 mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3327                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3328                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3329                                 NUM_BANKS(ADDR_SURF_8_BANK));
3330                 mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3331                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3332                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3333                                 NUM_BANKS(ADDR_SURF_8_BANK));
3334                 mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3335                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3336                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3337                                 NUM_BANKS(ADDR_SURF_8_BANK));
3338                 mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3339                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3340                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3341                                 NUM_BANKS(ADDR_SURF_8_BANK));
3342                 mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3343                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3344                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3345                                 NUM_BANKS(ADDR_SURF_8_BANK));
3346                 mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3347                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3348                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3349                                 NUM_BANKS(ADDR_SURF_8_BANK));
3350                 mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3351                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3352                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3353                                 NUM_BANKS(ADDR_SURF_8_BANK));
3354                 mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3355                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
3356                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3357                                 NUM_BANKS(ADDR_SURF_16_BANK));
3358                 mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3359                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3360                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3361                                 NUM_BANKS(ADDR_SURF_16_BANK));
3362                 mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3363                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3364                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3365                                  NUM_BANKS(ADDR_SURF_16_BANK));
3366                 mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3367                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3368                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3369                                  NUM_BANKS(ADDR_SURF_16_BANK));
3370                 mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3371                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3372                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3373                                  NUM_BANKS(ADDR_SURF_16_BANK));
3374                 mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3375                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3376                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3377                                  NUM_BANKS(ADDR_SURF_16_BANK));
3378                 mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3379                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3380                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3381                                  NUM_BANKS(ADDR_SURF_8_BANK));
3382
3383                 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
3384                         if (reg_offset != 7 && reg_offset != 12 && reg_offset != 17 &&
3385                             reg_offset != 23)
3386                                 WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]);
3387
3388                 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
3389                         if (reg_offset != 7)
3390                                 WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]);
3391
3392                 break;
3393         }
3394 }
3395
3396 static void gfx_v8_0_select_se_sh(struct amdgpu_device *adev,
3397                                   u32 se_num, u32 sh_num, u32 instance)
3398 {
3399         u32 data;
3400
3401         if (instance == 0xffffffff)
3402                 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES, 1);
3403         else
3404                 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX, instance);
3405
3406         if (se_num == 0xffffffff)
3407                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES, 1);
3408         else
3409                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num);
3410
3411         if (sh_num == 0xffffffff)
3412                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_BROADCAST_WRITES, 1);
3413         else
3414                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_INDEX, sh_num);
3415
3416         WREG32(mmGRBM_GFX_INDEX, data);
3417 }
3418
3419 static void gfx_v8_0_select_me_pipe_q(struct amdgpu_device *adev,
3420                                   u32 me, u32 pipe, u32 q, u32 vm)
3421 {
3422         vi_srbm_select(adev, me, pipe, q, vm);
3423 }
3424
3425 static u32 gfx_v8_0_get_rb_active_bitmap(struct amdgpu_device *adev)
3426 {
3427         u32 data, mask;
3428
3429         data =  RREG32(mmCC_RB_BACKEND_DISABLE) |
3430                 RREG32(mmGC_USER_RB_BACKEND_DISABLE);
3431
3432         data = REG_GET_FIELD(data, GC_USER_RB_BACKEND_DISABLE, BACKEND_DISABLE);
3433
3434         mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se /
3435                                          adev->gfx.config.max_sh_per_se);
3436
3437         return (~data) & mask;
3438 }
3439
3440 static void
3441 gfx_v8_0_raster_config(struct amdgpu_device *adev, u32 *rconf, u32 *rconf1)
3442 {
3443         switch (adev->asic_type) {
3444         case CHIP_FIJI:
3445         case CHIP_VEGAM:
3446                 *rconf |= RB_MAP_PKR0(2) | RB_MAP_PKR1(2) |
3447                           RB_XSEL2(1) | PKR_MAP(2) |
3448                           PKR_XSEL(1) | PKR_YSEL(1) |
3449                           SE_MAP(2) | SE_XSEL(2) | SE_YSEL(3);
3450                 *rconf1 |= SE_PAIR_MAP(2) | SE_PAIR_XSEL(3) |
3451                            SE_PAIR_YSEL(2);
3452                 break;
3453         case CHIP_TONGA:
3454         case CHIP_POLARIS10:
3455                 *rconf |= RB_MAP_PKR0(2) | RB_XSEL2(1) | SE_MAP(2) |
3456                           SE_XSEL(1) | SE_YSEL(1);
3457                 *rconf1 |= SE_PAIR_MAP(2) | SE_PAIR_XSEL(2) |
3458                            SE_PAIR_YSEL(2);
3459                 break;
3460         case CHIP_TOPAZ:
3461         case CHIP_CARRIZO:
3462                 *rconf |= RB_MAP_PKR0(2);
3463                 *rconf1 |= 0x0;
3464                 break;
3465         case CHIP_POLARIS11:
3466         case CHIP_POLARIS12:
3467                 *rconf |= RB_MAP_PKR0(2) | RB_XSEL2(1) | SE_MAP(2) |
3468                           SE_XSEL(1) | SE_YSEL(1);
3469                 *rconf1 |= 0x0;
3470                 break;
3471         case CHIP_STONEY:
3472                 *rconf |= 0x0;
3473                 *rconf1 |= 0x0;
3474                 break;
3475         default:
3476                 DRM_ERROR("unknown asic: 0x%x\n", adev->asic_type);
3477                 break;
3478         }
3479 }
3480
3481 static void
3482 gfx_v8_0_write_harvested_raster_configs(struct amdgpu_device *adev,
3483                                         u32 raster_config, u32 raster_config_1,
3484                                         unsigned rb_mask, unsigned num_rb)
3485 {
3486         unsigned sh_per_se = max_t(unsigned, adev->gfx.config.max_sh_per_se, 1);
3487         unsigned num_se = max_t(unsigned, adev->gfx.config.max_shader_engines, 1);
3488         unsigned rb_per_pkr = min_t(unsigned, num_rb / num_se / sh_per_se, 2);
3489         unsigned rb_per_se = num_rb / num_se;
3490         unsigned se_mask[4];
3491         unsigned se;
3492
3493         se_mask[0] = ((1 << rb_per_se) - 1) & rb_mask;
3494         se_mask[1] = (se_mask[0] << rb_per_se) & rb_mask;
3495         se_mask[2] = (se_mask[1] << rb_per_se) & rb_mask;
3496         se_mask[3] = (se_mask[2] << rb_per_se) & rb_mask;
3497
3498         WARN_ON(!(num_se == 1 || num_se == 2 || num_se == 4));
3499         WARN_ON(!(sh_per_se == 1 || sh_per_se == 2));
3500         WARN_ON(!(rb_per_pkr == 1 || rb_per_pkr == 2));
3501
3502         if ((num_se > 2) && ((!se_mask[0] && !se_mask[1]) ||
3503                              (!se_mask[2] && !se_mask[3]))) {
3504                 raster_config_1 &= ~SE_PAIR_MAP_MASK;
3505
3506                 if (!se_mask[0] && !se_mask[1]) {
3507                         raster_config_1 |=
3508                                 SE_PAIR_MAP(RASTER_CONFIG_SE_PAIR_MAP_3);
3509                 } else {
3510                         raster_config_1 |=
3511                                 SE_PAIR_MAP(RASTER_CONFIG_SE_PAIR_MAP_0);
3512                 }
3513         }
3514
3515         for (se = 0; se < num_se; se++) {
3516                 unsigned raster_config_se = raster_config;
3517                 unsigned pkr0_mask = ((1 << rb_per_pkr) - 1) << (se * rb_per_se);
3518                 unsigned pkr1_mask = pkr0_mask << rb_per_pkr;
3519                 int idx = (se / 2) * 2;
3520
3521                 if ((num_se > 1) && (!se_mask[idx] || !se_mask[idx + 1])) {
3522                         raster_config_se &= ~SE_MAP_MASK;
3523
3524                         if (!se_mask[idx]) {
3525                                 raster_config_se |= SE_MAP(RASTER_CONFIG_SE_MAP_3);
3526                         } else {
3527                                 raster_config_se |= SE_MAP(RASTER_CONFIG_SE_MAP_0);
3528                         }
3529                 }
3530
3531                 pkr0_mask &= rb_mask;
3532                 pkr1_mask &= rb_mask;
3533                 if (rb_per_se > 2 && (!pkr0_mask || !pkr1_mask)) {
3534                         raster_config_se &= ~PKR_MAP_MASK;
3535
3536                         if (!pkr0_mask) {
3537                                 raster_config_se |= PKR_MAP(RASTER_CONFIG_PKR_MAP_3);
3538                         } else {
3539                                 raster_config_se |= PKR_MAP(RASTER_CONFIG_PKR_MAP_0);
3540                         }
3541                 }
3542
3543                 if (rb_per_se >= 2) {
3544                         unsigned rb0_mask = 1 << (se * rb_per_se);
3545                         unsigned rb1_mask = rb0_mask << 1;
3546
3547                         rb0_mask &= rb_mask;
3548                         rb1_mask &= rb_mask;
3549                         if (!rb0_mask || !rb1_mask) {
3550                                 raster_config_se &= ~RB_MAP_PKR0_MASK;
3551
3552                                 if (!rb0_mask) {
3553                                         raster_config_se |=
3554                                                 RB_MAP_PKR0(RASTER_CONFIG_RB_MAP_3);
3555                                 } else {
3556                                         raster_config_se |=
3557                                                 RB_MAP_PKR0(RASTER_CONFIG_RB_MAP_0);
3558                                 }
3559                         }
3560
3561                         if (rb_per_se > 2) {
3562                                 rb0_mask = 1 << (se * rb_per_se + rb_per_pkr);
3563                                 rb1_mask = rb0_mask << 1;
3564                                 rb0_mask &= rb_mask;
3565                                 rb1_mask &= rb_mask;
3566                                 if (!rb0_mask || !rb1_mask) {
3567                                         raster_config_se &= ~RB_MAP_PKR1_MASK;
3568
3569                                         if (!rb0_mask) {
3570                                                 raster_config_se |=
3571                                                         RB_MAP_PKR1(RASTER_CONFIG_RB_MAP_3);
3572                                         } else {
3573                                                 raster_config_se |=
3574                                                         RB_MAP_PKR1(RASTER_CONFIG_RB_MAP_0);
3575                                         }
3576                                 }
3577                         }
3578                 }
3579
3580                 /* GRBM_GFX_INDEX has a different offset on VI */
3581                 gfx_v8_0_select_se_sh(adev, se, 0xffffffff, 0xffffffff);
3582                 WREG32(mmPA_SC_RASTER_CONFIG, raster_config_se);
3583                 WREG32(mmPA_SC_RASTER_CONFIG_1, raster_config_1);
3584         }
3585
3586         /* GRBM_GFX_INDEX has a different offset on VI */
3587         gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
3588 }
3589
3590 static void gfx_v8_0_setup_rb(struct amdgpu_device *adev)
3591 {
3592         int i, j;
3593         u32 data;
3594         u32 raster_config = 0, raster_config_1 = 0;
3595         u32 active_rbs = 0;
3596         u32 rb_bitmap_width_per_sh = adev->gfx.config.max_backends_per_se /
3597                                         adev->gfx.config.max_sh_per_se;
3598         unsigned num_rb_pipes;
3599
3600         mutex_lock(&adev->grbm_idx_mutex);
3601         for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
3602                 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
3603                         gfx_v8_0_select_se_sh(adev, i, j, 0xffffffff);
3604                         data = gfx_v8_0_get_rb_active_bitmap(adev);
3605                         active_rbs |= data << ((i * adev->gfx.config.max_sh_per_se + j) *
3606                                                rb_bitmap_width_per_sh);
3607                 }
3608         }
3609         gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
3610
3611         adev->gfx.config.backend_enable_mask = active_rbs;
3612         adev->gfx.config.num_rbs = hweight32(active_rbs);
3613
3614         num_rb_pipes = min_t(unsigned, adev->gfx.config.max_backends_per_se *
3615                              adev->gfx.config.max_shader_engines, 16);
3616
3617         gfx_v8_0_raster_config(adev, &raster_config, &raster_config_1);
3618
3619         if (!adev->gfx.config.backend_enable_mask ||
3620                         adev->gfx.config.num_rbs >= num_rb_pipes) {
3621                 WREG32(mmPA_SC_RASTER_CONFIG, raster_config);
3622                 WREG32(mmPA_SC_RASTER_CONFIG_1, raster_config_1);
3623         } else {
3624                 gfx_v8_0_write_harvested_raster_configs(adev, raster_config, raster_config_1,
3625                                                         adev->gfx.config.backend_enable_mask,
3626                                                         num_rb_pipes);
3627         }
3628
3629         /* cache the values for userspace */
3630         for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
3631                 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
3632                         gfx_v8_0_select_se_sh(adev, i, j, 0xffffffff);
3633                         adev->gfx.config.rb_config[i][j].rb_backend_disable =
3634                                 RREG32(mmCC_RB_BACKEND_DISABLE);
3635                         adev->gfx.config.rb_config[i][j].user_rb_backend_disable =
3636                                 RREG32(mmGC_USER_RB_BACKEND_DISABLE);
3637                         adev->gfx.config.rb_config[i][j].raster_config =
3638                                 RREG32(mmPA_SC_RASTER_CONFIG);
3639                         adev->gfx.config.rb_config[i][j].raster_config_1 =
3640                                 RREG32(mmPA_SC_RASTER_CONFIG_1);
3641                 }
3642         }
3643         gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
3644         mutex_unlock(&adev->grbm_idx_mutex);
3645 }
3646
3647 #define DEFAULT_SH_MEM_BASES    (0x6000)
3648 /**
3649  * gfx_v8_0_init_compute_vmid - gart enable
3650  *
3651  * @adev: amdgpu_device pointer
3652  *
3653  * Initialize compute vmid sh_mem registers
3654  *
3655  */
3656 static void gfx_v8_0_init_compute_vmid(struct amdgpu_device *adev)
3657 {
3658         int i;
3659         uint32_t sh_mem_config;
3660         uint32_t sh_mem_bases;
3661
3662         /*
3663          * Configure apertures:
3664          * LDS:         0x60000000'00000000 - 0x60000001'00000000 (4GB)
3665          * Scratch:     0x60000001'00000000 - 0x60000002'00000000 (4GB)
3666          * GPUVM:       0x60010000'00000000 - 0x60020000'00000000 (1TB)
3667          */
3668         sh_mem_bases = DEFAULT_SH_MEM_BASES | (DEFAULT_SH_MEM_BASES << 16);
3669
3670         sh_mem_config = SH_MEM_ADDRESS_MODE_HSA64 <<
3671                         SH_MEM_CONFIG__ADDRESS_MODE__SHIFT |
3672                         SH_MEM_ALIGNMENT_MODE_UNALIGNED <<
3673                         SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT |
3674                         MTYPE_CC << SH_MEM_CONFIG__DEFAULT_MTYPE__SHIFT |
3675                         SH_MEM_CONFIG__PRIVATE_ATC_MASK;
3676
3677         mutex_lock(&adev->srbm_mutex);
3678         for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
3679                 vi_srbm_select(adev, 0, 0, 0, i);
3680                 /* CP and shaders */
3681                 WREG32(mmSH_MEM_CONFIG, sh_mem_config);
3682                 WREG32(mmSH_MEM_APE1_BASE, 1);
3683                 WREG32(mmSH_MEM_APE1_LIMIT, 0);
3684                 WREG32(mmSH_MEM_BASES, sh_mem_bases);
3685         }
3686         vi_srbm_select(adev, 0, 0, 0, 0);
3687         mutex_unlock(&adev->srbm_mutex);
3688
3689         /* Initialize all compute VMIDs to have no GDS, GWS, or OA
3690            access. These should be enabled by FW for target VMIDs. */
3691         for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
3692                 WREG32(amdgpu_gds_reg_offset[i].mem_base, 0);
3693                 WREG32(amdgpu_gds_reg_offset[i].mem_size, 0);
3694                 WREG32(amdgpu_gds_reg_offset[i].gws, 0);
3695                 WREG32(amdgpu_gds_reg_offset[i].oa, 0);
3696         }
3697 }
3698
3699 static void gfx_v8_0_init_gds_vmid(struct amdgpu_device *adev)
3700 {
3701         int vmid;
3702
3703         /*
3704          * Initialize all compute and user-gfx VMIDs to have no GDS, GWS, or OA
3705          * access. Compute VMIDs should be enabled by FW for target VMIDs,
3706          * the driver can enable them for graphics. VMID0 should maintain
3707          * access so that HWS firmware can save/restore entries.
3708          */
3709         for (vmid = 1; vmid < AMDGPU_NUM_VMID; vmid++) {
3710                 WREG32(amdgpu_gds_reg_offset[vmid].mem_base, 0);
3711                 WREG32(amdgpu_gds_reg_offset[vmid].mem_size, 0);
3712                 WREG32(amdgpu_gds_reg_offset[vmid].gws, 0);
3713                 WREG32(amdgpu_gds_reg_offset[vmid].oa, 0);
3714         }
3715 }
3716
3717 static void gfx_v8_0_config_init(struct amdgpu_device *adev)
3718 {
3719         switch (adev->asic_type) {
3720         default:
3721                 adev->gfx.config.double_offchip_lds_buf = 1;
3722                 break;
3723         case CHIP_CARRIZO:
3724         case CHIP_STONEY:
3725                 adev->gfx.config.double_offchip_lds_buf = 0;
3726                 break;
3727         }
3728 }
3729
3730 static void gfx_v8_0_constants_init(struct amdgpu_device *adev)
3731 {
3732         u32 tmp, sh_static_mem_cfg;
3733         int i;
3734
3735         WREG32_FIELD(GRBM_CNTL, READ_TIMEOUT, 0xFF);
3736         WREG32(mmGB_ADDR_CONFIG, adev->gfx.config.gb_addr_config);
3737         WREG32(mmHDP_ADDR_CONFIG, adev->gfx.config.gb_addr_config);
3738         WREG32(mmDMIF_ADDR_CALC, adev->gfx.config.gb_addr_config);
3739
3740         gfx_v8_0_tiling_mode_table_init(adev);
3741         gfx_v8_0_setup_rb(adev);
3742         gfx_v8_0_get_cu_info(adev);
3743         gfx_v8_0_config_init(adev);
3744
3745         /* XXX SH_MEM regs */
3746         /* where to put LDS, scratch, GPUVM in FSA64 space */
3747         sh_static_mem_cfg = REG_SET_FIELD(0, SH_STATIC_MEM_CONFIG,
3748                                    SWIZZLE_ENABLE, 1);
3749         sh_static_mem_cfg = REG_SET_FIELD(sh_static_mem_cfg, SH_STATIC_MEM_CONFIG,
3750                                    ELEMENT_SIZE, 1);
3751         sh_static_mem_cfg = REG_SET_FIELD(sh_static_mem_cfg, SH_STATIC_MEM_CONFIG,
3752                                    INDEX_STRIDE, 3);
3753         WREG32(mmSH_STATIC_MEM_CONFIG, sh_static_mem_cfg);
3754
3755         mutex_lock(&adev->srbm_mutex);
3756         for (i = 0; i < adev->vm_manager.id_mgr[0].num_ids; i++) {
3757                 vi_srbm_select(adev, 0, 0, 0, i);
3758                 /* CP and shaders */
3759                 if (i == 0) {
3760                         tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, DEFAULT_MTYPE, MTYPE_UC);
3761                         tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, APE1_MTYPE, MTYPE_UC);
3762                         tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, ALIGNMENT_MODE,
3763                                             SH_MEM_ALIGNMENT_MODE_UNALIGNED);
3764                         WREG32(mmSH_MEM_CONFIG, tmp);
3765                         WREG32(mmSH_MEM_BASES, 0);
3766                 } else {
3767                         tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, DEFAULT_MTYPE, MTYPE_NC);
3768                         tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, APE1_MTYPE, MTYPE_UC);
3769                         tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, ALIGNMENT_MODE,
3770                                             SH_MEM_ALIGNMENT_MODE_UNALIGNED);
3771                         WREG32(mmSH_MEM_CONFIG, tmp);
3772                         tmp = adev->gmc.shared_aperture_start >> 48;
3773                         WREG32(mmSH_MEM_BASES, tmp);
3774                 }
3775
3776                 WREG32(mmSH_MEM_APE1_BASE, 1);
3777                 WREG32(mmSH_MEM_APE1_LIMIT, 0);
3778         }
3779         vi_srbm_select(adev, 0, 0, 0, 0);
3780         mutex_unlock(&adev->srbm_mutex);
3781
3782         gfx_v8_0_init_compute_vmid(adev);
3783         gfx_v8_0_init_gds_vmid(adev);
3784
3785         mutex_lock(&adev->grbm_idx_mutex);
3786         /*
3787          * making sure that the following register writes will be broadcasted
3788          * to all the shaders
3789          */
3790         gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
3791
3792         WREG32(mmPA_SC_FIFO_SIZE,
3793                    (adev->gfx.config.sc_prim_fifo_size_frontend <<
3794                         PA_SC_FIFO_SIZE__SC_FRONTEND_PRIM_FIFO_SIZE__SHIFT) |
3795                    (adev->gfx.config.sc_prim_fifo_size_backend <<
3796                         PA_SC_FIFO_SIZE__SC_BACKEND_PRIM_FIFO_SIZE__SHIFT) |
3797                    (adev->gfx.config.sc_hiz_tile_fifo_size <<
3798                         PA_SC_FIFO_SIZE__SC_HIZ_TILE_FIFO_SIZE__SHIFT) |
3799                    (adev->gfx.config.sc_earlyz_tile_fifo_size <<
3800                         PA_SC_FIFO_SIZE__SC_EARLYZ_TILE_FIFO_SIZE__SHIFT));
3801
3802         tmp = RREG32(mmSPI_ARB_PRIORITY);
3803         tmp = REG_SET_FIELD(tmp, SPI_ARB_PRIORITY, PIPE_ORDER_TS0, 2);
3804         tmp = REG_SET_FIELD(tmp, SPI_ARB_PRIORITY, PIPE_ORDER_TS1, 2);
3805         tmp = REG_SET_FIELD(tmp, SPI_ARB_PRIORITY, PIPE_ORDER_TS2, 2);
3806         tmp = REG_SET_FIELD(tmp, SPI_ARB_PRIORITY, PIPE_ORDER_TS3, 2);
3807         WREG32(mmSPI_ARB_PRIORITY, tmp);
3808
3809         mutex_unlock(&adev->grbm_idx_mutex);
3810
3811 }
3812
3813 static void gfx_v8_0_wait_for_rlc_serdes(struct amdgpu_device *adev)
3814 {
3815         u32 i, j, k;
3816         u32 mask;
3817
3818         mutex_lock(&adev->grbm_idx_mutex);
3819         for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
3820                 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
3821                         gfx_v8_0_select_se_sh(adev, i, j, 0xffffffff);
3822                         for (k = 0; k < adev->usec_timeout; k++) {
3823                                 if (RREG32(mmRLC_SERDES_CU_MASTER_BUSY) == 0)
3824                                         break;
3825                                 udelay(1);
3826                         }
3827                         if (k == adev->usec_timeout) {
3828                                 gfx_v8_0_select_se_sh(adev, 0xffffffff,
3829                                                       0xffffffff, 0xffffffff);
3830                                 mutex_unlock(&adev->grbm_idx_mutex);
3831                                 DRM_INFO("Timeout wait for RLC serdes %u,%u\n",
3832                                          i, j);
3833                                 return;
3834                         }
3835                 }
3836         }
3837         gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
3838         mutex_unlock(&adev->grbm_idx_mutex);
3839
3840         mask = RLC_SERDES_NONCU_MASTER_BUSY__SE_MASTER_BUSY_MASK |
3841                 RLC_SERDES_NONCU_MASTER_BUSY__GC_MASTER_BUSY_MASK |
3842                 RLC_SERDES_NONCU_MASTER_BUSY__TC0_MASTER_BUSY_MASK |
3843                 RLC_SERDES_NONCU_MASTER_BUSY__TC1_MASTER_BUSY_MASK;
3844         for (k = 0; k < adev->usec_timeout; k++) {
3845                 if ((RREG32(mmRLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
3846                         break;
3847                 udelay(1);
3848         }
3849 }
3850
3851 static void gfx_v8_0_enable_gui_idle_interrupt(struct amdgpu_device *adev,
3852                                                bool enable)
3853 {
3854         u32 tmp = RREG32(mmCP_INT_CNTL_RING0);
3855
3856         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE, enable ? 1 : 0);
3857         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE, enable ? 1 : 0);
3858         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE, enable ? 1 : 0);
3859         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE, enable ? 1 : 0);
3860
3861         WREG32(mmCP_INT_CNTL_RING0, tmp);
3862 }
3863
3864 static void gfx_v8_0_init_csb(struct amdgpu_device *adev)
3865 {
3866         adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr);
3867         /* csib */
3868         WREG32(mmRLC_CSIB_ADDR_HI,
3869                         adev->gfx.rlc.clear_state_gpu_addr >> 32);
3870         WREG32(mmRLC_CSIB_ADDR_LO,
3871                         adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc);
3872         WREG32(mmRLC_CSIB_LENGTH,
3873                         adev->gfx.rlc.clear_state_size);
3874 }
3875
3876 static void gfx_v8_0_parse_ind_reg_list(int *register_list_format,
3877                                 int ind_offset,
3878                                 int list_size,
3879                                 int *unique_indices,
3880                                 int *indices_count,
3881                                 int max_indices,
3882                                 int *ind_start_offsets,
3883                                 int *offset_count,
3884                                 int max_offset)
3885 {
3886         int indices;
3887         bool new_entry = true;
3888
3889         for (; ind_offset < list_size; ind_offset++) {
3890
3891                 if (new_entry) {
3892                         new_entry = false;
3893                         ind_start_offsets[*offset_count] = ind_offset;
3894                         *offset_count = *offset_count + 1;
3895                         BUG_ON(*offset_count >= max_offset);
3896                 }
3897
3898                 if (register_list_format[ind_offset] == 0xFFFFFFFF) {
3899                         new_entry = true;
3900                         continue;
3901                 }
3902
3903                 ind_offset += 2;
3904
3905                 /* look for the matching indice */
3906                 for (indices = 0;
3907                         indices < *indices_count;
3908                         indices++) {
3909                         if (unique_indices[indices] ==
3910                                 register_list_format[ind_offset])
3911                                 break;
3912                 }
3913
3914                 if (indices >= *indices_count) {
3915                         unique_indices[*indices_count] =
3916                                 register_list_format[ind_offset];
3917                         indices = *indices_count;
3918                         *indices_count = *indices_count + 1;
3919                         BUG_ON(*indices_count >= max_indices);
3920                 }
3921
3922                 register_list_format[ind_offset] = indices;
3923         }
3924 }
3925
3926 static int gfx_v8_0_init_save_restore_list(struct amdgpu_device *adev)
3927 {
3928         int i, temp, data;
3929         int unique_indices[] = {0, 0, 0, 0, 0, 0, 0, 0};
3930         int indices_count = 0;
3931         int indirect_start_offsets[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
3932         int offset_count = 0;
3933
3934         int list_size;
3935         unsigned int *register_list_format =
3936                 kmemdup(adev->gfx.rlc.register_list_format,
3937                         adev->gfx.rlc.reg_list_format_size_bytes, GFP_KERNEL);
3938         if (!register_list_format)
3939                 return -ENOMEM;
3940
3941         gfx_v8_0_parse_ind_reg_list(register_list_format,
3942                                 RLC_FormatDirectRegListLength,
3943                                 adev->gfx.rlc.reg_list_format_size_bytes >> 2,
3944                                 unique_indices,
3945                                 &indices_count,
3946                                 ARRAY_SIZE(unique_indices),
3947                                 indirect_start_offsets,
3948                                 &offset_count,
3949                                 ARRAY_SIZE(indirect_start_offsets));
3950
3951         /* save and restore list */
3952         WREG32_FIELD(RLC_SRM_CNTL, AUTO_INCR_ADDR, 1);
3953
3954         WREG32(mmRLC_SRM_ARAM_ADDR, 0);
3955         for (i = 0; i < adev->gfx.rlc.reg_list_size_bytes >> 2; i++)
3956                 WREG32(mmRLC_SRM_ARAM_DATA, adev->gfx.rlc.register_restore[i]);
3957
3958         /* indirect list */
3959         WREG32(mmRLC_GPM_SCRATCH_ADDR, adev->gfx.rlc.reg_list_format_start);
3960         for (i = 0; i < adev->gfx.rlc.reg_list_format_size_bytes >> 2; i++)
3961                 WREG32(mmRLC_GPM_SCRATCH_DATA, register_list_format[i]);
3962
3963         list_size = adev->gfx.rlc.reg_list_size_bytes >> 2;
3964         list_size = list_size >> 1;
3965         WREG32(mmRLC_GPM_SCRATCH_ADDR, adev->gfx.rlc.reg_restore_list_size);
3966         WREG32(mmRLC_GPM_SCRATCH_DATA, list_size);
3967
3968         /* starting offsets starts */
3969         WREG32(mmRLC_GPM_SCRATCH_ADDR,
3970                 adev->gfx.rlc.starting_offsets_start);
3971         for (i = 0; i < ARRAY_SIZE(indirect_start_offsets); i++)
3972                 WREG32(mmRLC_GPM_SCRATCH_DATA,
3973                                 indirect_start_offsets[i]);
3974
3975         /* unique indices */
3976         temp = mmRLC_SRM_INDEX_CNTL_ADDR_0;
3977         data = mmRLC_SRM_INDEX_CNTL_DATA_0;
3978         for (i = 0; i < ARRAY_SIZE(unique_indices); i++) {
3979                 if (unique_indices[i] != 0) {
3980                         WREG32(temp + i, unique_indices[i] & 0x3FFFF);
3981                         WREG32(data + i, unique_indices[i] >> 20);
3982                 }
3983         }
3984         kfree(register_list_format);
3985
3986         return 0;
3987 }
3988
3989 static void gfx_v8_0_enable_save_restore_machine(struct amdgpu_device *adev)
3990 {
3991         WREG32_FIELD(RLC_SRM_CNTL, SRM_ENABLE, 1);
3992 }
3993
3994 static void gfx_v8_0_init_power_gating(struct amdgpu_device *adev)
3995 {
3996         uint32_t data;
3997
3998         WREG32_FIELD(CP_RB_WPTR_POLL_CNTL, IDLE_POLL_COUNT, 0x60);
3999
4000         data = REG_SET_FIELD(0, RLC_PG_DELAY, POWER_UP_DELAY, 0x10);
4001         data = REG_SET_FIELD(data, RLC_PG_DELAY, POWER_DOWN_DELAY, 0x10);
4002         data = REG_SET_FIELD(data, RLC_PG_DELAY, CMD_PROPAGATE_DELAY, 0x10);
4003         data = REG_SET_FIELD(data, RLC_PG_DELAY, MEM_SLEEP_DELAY, 0x10);
4004         WREG32(mmRLC_PG_DELAY, data);
4005
4006         WREG32_FIELD(RLC_PG_DELAY_2, SERDES_CMD_DELAY, 0x3);
4007         WREG32_FIELD(RLC_AUTO_PG_CTRL, GRBM_REG_SAVE_GFX_IDLE_THRESHOLD, 0x55f0);
4008
4009 }
4010
4011 static void cz_enable_sck_slow_down_on_power_up(struct amdgpu_device *adev,
4012                                                 bool enable)
4013 {
4014         WREG32_FIELD(RLC_PG_CNTL, SMU_CLK_SLOWDOWN_ON_PU_ENABLE, enable ? 1 : 0);
4015 }
4016
4017 static void cz_enable_sck_slow_down_on_power_down(struct amdgpu_device *adev,
4018                                                   bool enable)
4019 {
4020         WREG32_FIELD(RLC_PG_CNTL, SMU_CLK_SLOWDOWN_ON_PD_ENABLE, enable ? 1 : 0);
4021 }
4022
4023 static void cz_enable_cp_power_gating(struct amdgpu_device *adev, bool enable)
4024 {
4025         WREG32_FIELD(RLC_PG_CNTL, CP_PG_DISABLE, enable ? 0 : 1);
4026 }
4027
4028 static void gfx_v8_0_init_pg(struct amdgpu_device *adev)
4029 {
4030         if ((adev->asic_type == CHIP_CARRIZO) ||
4031             (adev->asic_type == CHIP_STONEY)) {
4032                 gfx_v8_0_init_csb(adev);
4033                 gfx_v8_0_init_save_restore_list(adev);
4034                 gfx_v8_0_enable_save_restore_machine(adev);
4035                 WREG32(mmRLC_JUMP_TABLE_RESTORE, adev->gfx.rlc.cp_table_gpu_addr >> 8);
4036                 gfx_v8_0_init_power_gating(adev);
4037                 WREG32(mmRLC_PG_ALWAYS_ON_CU_MASK, adev->gfx.cu_info.ao_cu_mask);
4038         } else if ((adev->asic_type == CHIP_POLARIS11) ||
4039                    (adev->asic_type == CHIP_POLARIS12) ||
4040                    (adev->asic_type == CHIP_VEGAM)) {
4041                 gfx_v8_0_init_csb(adev);
4042                 gfx_v8_0_init_save_restore_list(adev);
4043                 gfx_v8_0_enable_save_restore_machine(adev);
4044                 gfx_v8_0_init_power_gating(adev);
4045         }
4046
4047 }
4048
4049 static void gfx_v8_0_rlc_stop(struct amdgpu_device *adev)
4050 {
4051         WREG32_FIELD(RLC_CNTL, RLC_ENABLE_F32, 0);
4052
4053         gfx_v8_0_enable_gui_idle_interrupt(adev, false);
4054         gfx_v8_0_wait_for_rlc_serdes(adev);
4055 }
4056
4057 static void gfx_v8_0_rlc_reset(struct amdgpu_device *adev)
4058 {
4059         WREG32_FIELD(GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
4060         udelay(50);
4061
4062         WREG32_FIELD(GRBM_SOFT_RESET, SOFT_RESET_RLC, 0);
4063         udelay(50);
4064 }
4065
4066 static void gfx_v8_0_rlc_start(struct amdgpu_device *adev)
4067 {
4068         WREG32_FIELD(RLC_CNTL, RLC_ENABLE_F32, 1);
4069
4070         /* carrizo do enable cp interrupt after cp inited */
4071         if (!(adev->flags & AMD_IS_APU))
4072                 gfx_v8_0_enable_gui_idle_interrupt(adev, true);
4073
4074         udelay(50);
4075 }
4076
4077 static int gfx_v8_0_rlc_resume(struct amdgpu_device *adev)
4078 {
4079         if (amdgpu_sriov_vf(adev)) {
4080                 gfx_v8_0_init_csb(adev);
4081                 return 0;
4082         }
4083
4084         adev->gfx.rlc.funcs->stop(adev);
4085         adev->gfx.rlc.funcs->reset(adev);
4086         gfx_v8_0_init_pg(adev);
4087         adev->gfx.rlc.funcs->start(adev);
4088
4089         return 0;
4090 }
4091
4092 static void gfx_v8_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable)
4093 {
4094         u32 tmp = RREG32(mmCP_ME_CNTL);
4095
4096         if (enable) {
4097                 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, 0);
4098                 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, 0);
4099                 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_HALT, 0);
4100         } else {
4101                 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, 1);
4102                 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, 1);
4103                 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_HALT, 1);
4104         }
4105         WREG32(mmCP_ME_CNTL, tmp);
4106         udelay(50);
4107 }
4108
4109 static u32 gfx_v8_0_get_csb_size(struct amdgpu_device *adev)
4110 {
4111         u32 count = 0;
4112         const struct cs_section_def *sect = NULL;
4113         const struct cs_extent_def *ext = NULL;
4114
4115         /* begin clear state */
4116         count += 2;
4117         /* context control state */
4118         count += 3;
4119
4120         for (sect = vi_cs_data; sect->section != NULL; ++sect) {
4121                 for (ext = sect->section; ext->extent != NULL; ++ext) {
4122                         if (sect->id == SECT_CONTEXT)
4123                                 count += 2 + ext->reg_count;
4124                         else
4125                                 return 0;
4126                 }
4127         }
4128         /* pa_sc_raster_config/pa_sc_raster_config1 */
4129         count += 4;
4130         /* end clear state */
4131         count += 2;
4132         /* clear state */
4133         count += 2;
4134
4135         return count;
4136 }
4137
4138 static int gfx_v8_0_cp_gfx_start(struct amdgpu_device *adev)
4139 {
4140         struct amdgpu_ring *ring = &adev->gfx.gfx_ring[0];
4141         const struct cs_section_def *sect = NULL;
4142         const struct cs_extent_def *ext = NULL;
4143         int r, i;
4144
4145         /* init the CP */
4146         WREG32(mmCP_MAX_CONTEXT, adev->gfx.config.max_hw_contexts - 1);
4147         WREG32(mmCP_ENDIAN_SWAP, 0);
4148         WREG32(mmCP_DEVICE_ID, 1);
4149
4150         gfx_v8_0_cp_gfx_enable(adev, true);
4151
4152         r = amdgpu_ring_alloc(ring, gfx_v8_0_get_csb_size(adev) + 4);
4153         if (r) {
4154                 DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r);
4155                 return r;
4156         }
4157
4158         /* clear state buffer */
4159         amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4160         amdgpu_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
4161
4162         amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
4163         amdgpu_ring_write(ring, 0x80000000);
4164         amdgpu_ring_write(ring, 0x80000000);
4165
4166         for (sect = vi_cs_data; sect->section != NULL; ++sect) {
4167                 for (ext = sect->section; ext->extent != NULL; ++ext) {
4168                         if (sect->id == SECT_CONTEXT) {
4169                                 amdgpu_ring_write(ring,
4170                                        PACKET3(PACKET3_SET_CONTEXT_REG,
4171                                                ext->reg_count));
4172                                 amdgpu_ring_write(ring,
4173                                        ext->reg_index - PACKET3_SET_CONTEXT_REG_START);
4174                                 for (i = 0; i < ext->reg_count; i++)
4175                                         amdgpu_ring_write(ring, ext->extent[i]);
4176                         }
4177                 }
4178         }
4179
4180         amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
4181         amdgpu_ring_write(ring, mmPA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
4182         amdgpu_ring_write(ring, adev->gfx.config.rb_config[0][0].raster_config);
4183         amdgpu_ring_write(ring, adev->gfx.config.rb_config[0][0].raster_config_1);
4184
4185         amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4186         amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
4187
4188         amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
4189         amdgpu_ring_write(ring, 0);
4190
4191         /* init the CE partitions */
4192         amdgpu_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
4193         amdgpu_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
4194         amdgpu_ring_write(ring, 0x8000);
4195         amdgpu_ring_write(ring, 0x8000);
4196
4197         amdgpu_ring_commit(ring);
4198
4199         return 0;
4200 }
4201 static void gfx_v8_0_set_cpg_door_bell(struct amdgpu_device *adev, struct amdgpu_ring *ring)
4202 {
4203         u32 tmp;
4204         /* no gfx doorbells on iceland */
4205         if (adev->asic_type == CHIP_TOPAZ)
4206                 return;
4207
4208         tmp = RREG32(mmCP_RB_DOORBELL_CONTROL);
4209
4210         if (ring->use_doorbell) {
4211                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
4212                                 DOORBELL_OFFSET, ring->doorbell_index);
4213                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
4214                                                 DOORBELL_HIT, 0);
4215                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
4216                                             DOORBELL_EN, 1);
4217         } else {
4218                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, DOORBELL_EN, 0);
4219         }
4220
4221         WREG32(mmCP_RB_DOORBELL_CONTROL, tmp);
4222
4223         if (adev->flags & AMD_IS_APU)
4224                 return;
4225
4226         tmp = REG_SET_FIELD(0, CP_RB_DOORBELL_RANGE_LOWER,
4227                                         DOORBELL_RANGE_LOWER,
4228                                         adev->doorbell_index.gfx_ring0);
4229         WREG32(mmCP_RB_DOORBELL_RANGE_LOWER, tmp);
4230
4231         WREG32(mmCP_RB_DOORBELL_RANGE_UPPER,
4232                 CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK);
4233 }
4234
4235 static int gfx_v8_0_cp_gfx_resume(struct amdgpu_device *adev)
4236 {
4237         struct amdgpu_ring *ring;
4238         u32 tmp;
4239         u32 rb_bufsz;
4240         u64 rb_addr, rptr_addr, wptr_gpu_addr;
4241
4242         /* Set the write pointer delay */
4243         WREG32(mmCP_RB_WPTR_DELAY, 0);
4244
4245         /* set the RB to use vmid 0 */
4246         WREG32(mmCP_RB_VMID, 0);
4247
4248         /* Set ring buffer size */
4249         ring = &adev->gfx.gfx_ring[0];
4250         rb_bufsz = order_base_2(ring->ring_size / 8);
4251         tmp = REG_SET_FIELD(0, CP_RB0_CNTL, RB_BUFSZ, rb_bufsz);
4252         tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, RB_BLKSZ, rb_bufsz - 2);
4253         tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, MTYPE, 3);
4254         tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, MIN_IB_AVAILSZ, 1);
4255 #ifdef __BIG_ENDIAN
4256         tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, BUF_SWAP, 1);
4257 #endif
4258         WREG32(mmCP_RB0_CNTL, tmp);
4259
4260         /* Initialize the ring buffer's read and write pointers */
4261         WREG32(mmCP_RB0_CNTL, tmp | CP_RB0_CNTL__RB_RPTR_WR_ENA_MASK);
4262         ring->wptr = 0;
4263         WREG32(mmCP_RB0_WPTR, lower_32_bits(ring->wptr));
4264
4265         /* set the wb address wether it's enabled or not */
4266         rptr_addr = ring->rptr_gpu_addr;
4267         WREG32(mmCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr));
4268         WREG32(mmCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & 0xFF);
4269
4270         wptr_gpu_addr = ring->wptr_gpu_addr;
4271         WREG32(mmCP_RB_WPTR_POLL_ADDR_LO, lower_32_bits(wptr_gpu_addr));
4272         WREG32(mmCP_RB_WPTR_POLL_ADDR_HI, upper_32_bits(wptr_gpu_addr));
4273         mdelay(1);
4274         WREG32(mmCP_RB0_CNTL, tmp);
4275
4276         rb_addr = ring->gpu_addr >> 8;
4277         WREG32(mmCP_RB0_BASE, rb_addr);
4278         WREG32(mmCP_RB0_BASE_HI, upper_32_bits(rb_addr));
4279
4280         gfx_v8_0_set_cpg_door_bell(adev, ring);
4281         /* start the ring */
4282         amdgpu_ring_clear_ring(ring);
4283         gfx_v8_0_cp_gfx_start(adev);
4284         ring->sched.ready = true;
4285
4286         return 0;
4287 }
4288
4289 static void gfx_v8_0_cp_compute_enable(struct amdgpu_device *adev, bool enable)
4290 {
4291         if (enable) {
4292                 WREG32(mmCP_MEC_CNTL, 0);
4293         } else {
4294                 WREG32(mmCP_MEC_CNTL, (CP_MEC_CNTL__MEC_ME1_HALT_MASK | CP_MEC_CNTL__MEC_ME2_HALT_MASK));
4295                 adev->gfx.kiq.ring.sched.ready = false;
4296         }
4297         udelay(50);
4298 }
4299
4300 /* KIQ functions */
4301 static void gfx_v8_0_kiq_setting(struct amdgpu_ring *ring)
4302 {
4303         uint32_t tmp;
4304         struct amdgpu_device *adev = ring->adev;
4305
4306         /* tell RLC which is KIQ queue */
4307         tmp = RREG32(mmRLC_CP_SCHEDULERS);
4308         tmp &= 0xffffff00;
4309         tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue);
4310         WREG32(mmRLC_CP_SCHEDULERS, tmp);
4311         tmp |= 0x80;
4312         WREG32(mmRLC_CP_SCHEDULERS, tmp);
4313 }
4314
4315 static int gfx_v8_0_kiq_kcq_enable(struct amdgpu_device *adev)
4316 {
4317         struct amdgpu_ring *kiq_ring = &adev->gfx.kiq.ring;
4318         uint64_t queue_mask = 0;
4319         int r, i;
4320
4321         for (i = 0; i < AMDGPU_MAX_COMPUTE_QUEUES; ++i) {
4322                 if (!test_bit(i, adev->gfx.mec.queue_bitmap))
4323                         continue;
4324
4325                 /* This situation may be hit in the future if a new HW
4326                  * generation exposes more than 64 queues. If so, the
4327                  * definition of queue_mask needs updating */
4328                 if (WARN_ON(i >= (sizeof(queue_mask)*8))) {
4329                         DRM_ERROR("Invalid KCQ enabled: %d\n", i);
4330                         break;
4331                 }
4332
4333                 queue_mask |= (1ull << i);
4334         }
4335
4336         r = amdgpu_ring_alloc(kiq_ring, (8 * adev->gfx.num_compute_rings) + 8);
4337         if (r) {
4338                 DRM_ERROR("Failed to lock KIQ (%d).\n", r);
4339                 return r;
4340         }
4341         /* set resources */
4342         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_RESOURCES, 6));
4343         amdgpu_ring_write(kiq_ring, 0); /* vmid_mask:0 queue_type:0 (KIQ) */
4344         amdgpu_ring_write(kiq_ring, lower_32_bits(queue_mask)); /* queue mask lo */
4345         amdgpu_ring_write(kiq_ring, upper_32_bits(queue_mask)); /* queue mask hi */
4346         amdgpu_ring_write(kiq_ring, 0); /* gws mask lo */
4347         amdgpu_ring_write(kiq_ring, 0); /* gws mask hi */
4348         amdgpu_ring_write(kiq_ring, 0); /* oac mask */
4349         amdgpu_ring_write(kiq_ring, 0); /* gds heap base:0, gds heap size:0 */
4350         for (i = 0; i < adev->gfx.num_compute_rings; i++) {
4351                 struct amdgpu_ring *ring = &adev->gfx.compute_ring[i];
4352                 uint64_t mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj);
4353                 uint64_t wptr_addr = ring->wptr_gpu_addr;
4354
4355                 /* map queues */
4356                 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5));
4357                 /* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/
4358                 amdgpu_ring_write(kiq_ring,
4359                                   PACKET3_MAP_QUEUES_NUM_QUEUES(1));
4360                 amdgpu_ring_write(kiq_ring,
4361                                   PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index) |
4362                                   PACKET3_MAP_QUEUES_QUEUE(ring->queue) |
4363                                   PACKET3_MAP_QUEUES_PIPE(ring->pipe) |
4364                                   PACKET3_MAP_QUEUES_ME(ring->me == 1 ? 0 : 1)); /* doorbell */
4365                 amdgpu_ring_write(kiq_ring, lower_32_bits(mqd_addr));
4366                 amdgpu_ring_write(kiq_ring, upper_32_bits(mqd_addr));
4367                 amdgpu_ring_write(kiq_ring, lower_32_bits(wptr_addr));
4368                 amdgpu_ring_write(kiq_ring, upper_32_bits(wptr_addr));
4369         }
4370
4371         amdgpu_ring_commit(kiq_ring);
4372
4373         return 0;
4374 }
4375
4376 static int gfx_v8_0_deactivate_hqd(struct amdgpu_device *adev, u32 req)
4377 {
4378         int i, r = 0;
4379
4380         if (RREG32(mmCP_HQD_ACTIVE) & CP_HQD_ACTIVE__ACTIVE_MASK) {
4381                 WREG32_FIELD(CP_HQD_DEQUEUE_REQUEST, DEQUEUE_REQ, req);
4382                 for (i = 0; i < adev->usec_timeout; i++) {
4383                         if (!(RREG32(mmCP_HQD_ACTIVE) & CP_HQD_ACTIVE__ACTIVE_MASK))
4384                                 break;
4385                         udelay(1);
4386                 }
4387                 if (i == adev->usec_timeout)
4388                         r = -ETIMEDOUT;
4389         }
4390         WREG32(mmCP_HQD_DEQUEUE_REQUEST, 0);
4391         WREG32(mmCP_HQD_PQ_RPTR, 0);
4392         WREG32(mmCP_HQD_PQ_WPTR, 0);
4393
4394         return r;
4395 }
4396
4397 static void gfx_v8_0_mqd_set_priority(struct amdgpu_ring *ring, struct vi_mqd *mqd)
4398 {
4399         struct amdgpu_device *adev = ring->adev;
4400
4401         if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
4402                 if (amdgpu_gfx_is_high_priority_compute_queue(adev, ring)) {
4403                         mqd->cp_hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_HIGH;
4404                         mqd->cp_hqd_queue_priority =
4405                                 AMDGPU_GFX_QUEUE_PRIORITY_MAXIMUM;
4406                 }
4407         }
4408 }
4409
4410 static int gfx_v8_0_mqd_init(struct amdgpu_ring *ring)
4411 {
4412         struct amdgpu_device *adev = ring->adev;
4413         struct vi_mqd *mqd = ring->mqd_ptr;
4414         uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr;
4415         uint32_t tmp;
4416
4417         mqd->header = 0xC0310800;
4418         mqd->compute_pipelinestat_enable = 0x00000001;
4419         mqd->compute_static_thread_mgmt_se0 = 0xffffffff;
4420         mqd->compute_static_thread_mgmt_se1 = 0xffffffff;
4421         mqd->compute_static_thread_mgmt_se2 = 0xffffffff;
4422         mqd->compute_static_thread_mgmt_se3 = 0xffffffff;
4423         mqd->compute_misc_reserved = 0x00000003;
4424         mqd->dynamic_cu_mask_addr_lo = lower_32_bits(ring->mqd_gpu_addr
4425                                                      + offsetof(struct vi_mqd_allocation, dynamic_cu_mask));
4426         mqd->dynamic_cu_mask_addr_hi = upper_32_bits(ring->mqd_gpu_addr
4427                                                      + offsetof(struct vi_mqd_allocation, dynamic_cu_mask));
4428         eop_base_addr = ring->eop_gpu_addr >> 8;
4429         mqd->cp_hqd_eop_base_addr_lo = eop_base_addr;
4430         mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr);
4431
4432         /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4433         tmp = RREG32(mmCP_HQD_EOP_CONTROL);
4434         tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE,
4435                         (order_base_2(GFX8_MEC_HPD_SIZE / 4) - 1));
4436
4437         mqd->cp_hqd_eop_control = tmp;
4438
4439         /* enable doorbell? */
4440         tmp = REG_SET_FIELD(RREG32(mmCP_HQD_PQ_DOORBELL_CONTROL),
4441                             CP_HQD_PQ_DOORBELL_CONTROL,
4442                             DOORBELL_EN,
4443                             ring->use_doorbell ? 1 : 0);
4444
4445         mqd->cp_hqd_pq_doorbell_control = tmp;
4446
4447         /* set the pointer to the MQD */
4448         mqd->cp_mqd_base_addr_lo = ring->mqd_gpu_addr & 0xfffffffc;
4449         mqd->cp_mqd_base_addr_hi = upper_32_bits(ring->mqd_gpu_addr);
4450
4451         /* set MQD vmid to 0 */
4452         tmp = RREG32(mmCP_MQD_CONTROL);
4453         tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0);
4454         mqd->cp_mqd_control = tmp;
4455
4456         /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
4457         hqd_gpu_addr = ring->gpu_addr >> 8;
4458         mqd->cp_hqd_pq_base_lo = hqd_gpu_addr;
4459         mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
4460
4461         /* set up the HQD, this is similar to CP_RB0_CNTL */
4462         tmp = RREG32(mmCP_HQD_PQ_CONTROL);
4463         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE,
4464                             (order_base_2(ring->ring_size / 4) - 1));
4465         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE,
4466                         (order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1));
4467 #ifdef __BIG_ENDIAN
4468         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ENDIAN_SWAP, 1);
4469 #endif
4470         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 0);
4471         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ROQ_PQ_IB_FLIP, 0);
4472         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1);
4473         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1);
4474         mqd->cp_hqd_pq_control = tmp;
4475
4476         /* set the wb address whether it's enabled or not */
4477         wb_gpu_addr = ring->rptr_gpu_addr;
4478         mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc;
4479         mqd->cp_hqd_pq_rptr_report_addr_hi =
4480                 upper_32_bits(wb_gpu_addr) & 0xffff;
4481
4482         /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
4483         wb_gpu_addr = ring->wptr_gpu_addr;
4484         mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc;
4485         mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
4486
4487         tmp = 0;
4488         /* enable the doorbell if requested */
4489         if (ring->use_doorbell) {
4490                 tmp = RREG32(mmCP_HQD_PQ_DOORBELL_CONTROL);
4491                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4492                                 DOORBELL_OFFSET, ring->doorbell_index);
4493
4494                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4495                                          DOORBELL_EN, 1);
4496                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4497                                          DOORBELL_SOURCE, 0);
4498                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4499                                          DOORBELL_HIT, 0);
4500         }
4501
4502         mqd->cp_hqd_pq_doorbell_control = tmp;
4503
4504         /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
4505         ring->wptr = 0;
4506         mqd->cp_hqd_pq_wptr = ring->wptr;
4507         mqd->cp_hqd_pq_rptr = RREG32(mmCP_HQD_PQ_RPTR);
4508
4509         /* set the vmid for the queue */
4510         mqd->cp_hqd_vmid = 0;
4511
4512         tmp = RREG32(mmCP_HQD_PERSISTENT_STATE);
4513         tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x53);
4514         mqd->cp_hqd_persistent_state = tmp;
4515
4516         /* set MTYPE */
4517         tmp = RREG32(mmCP_HQD_IB_CONTROL);
4518         tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 3);
4519         tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MTYPE, 3);
4520         mqd->cp_hqd_ib_control = tmp;
4521
4522         tmp = RREG32(mmCP_HQD_IQ_TIMER);
4523         tmp = REG_SET_FIELD(tmp, CP_HQD_IQ_TIMER, MTYPE, 3);
4524         mqd->cp_hqd_iq_timer = tmp;
4525
4526         tmp = RREG32(mmCP_HQD_CTX_SAVE_CONTROL);
4527         tmp = REG_SET_FIELD(tmp, CP_HQD_CTX_SAVE_CONTROL, MTYPE, 3);
4528         mqd->cp_hqd_ctx_save_control = tmp;
4529
4530         /* defaults */
4531         mqd->cp_hqd_eop_rptr = RREG32(mmCP_HQD_EOP_RPTR);
4532         mqd->cp_hqd_eop_wptr = RREG32(mmCP_HQD_EOP_WPTR);
4533         mqd->cp_hqd_ctx_save_base_addr_lo = RREG32(mmCP_HQD_CTX_SAVE_BASE_ADDR_LO);
4534         mqd->cp_hqd_ctx_save_base_addr_hi = RREG32(mmCP_HQD_CTX_SAVE_BASE_ADDR_HI);
4535         mqd->cp_hqd_cntl_stack_offset = RREG32(mmCP_HQD_CNTL_STACK_OFFSET);
4536         mqd->cp_hqd_cntl_stack_size = RREG32(mmCP_HQD_CNTL_STACK_SIZE);
4537         mqd->cp_hqd_wg_state_offset = RREG32(mmCP_HQD_WG_STATE_OFFSET);
4538         mqd->cp_hqd_ctx_save_size = RREG32(mmCP_HQD_CTX_SAVE_SIZE);
4539         mqd->cp_hqd_eop_done_events = RREG32(mmCP_HQD_EOP_EVENTS);
4540         mqd->cp_hqd_error = RREG32(mmCP_HQD_ERROR);
4541         mqd->cp_hqd_eop_wptr_mem = RREG32(mmCP_HQD_EOP_WPTR_MEM);
4542         mqd->cp_hqd_eop_dones = RREG32(mmCP_HQD_EOP_DONES);
4543
4544         /* set static priority for a queue/ring */
4545         gfx_v8_0_mqd_set_priority(ring, mqd);
4546         mqd->cp_hqd_quantum = RREG32(mmCP_HQD_QUANTUM);
4547
4548         /* map_queues packet doesn't need activate the queue,
4549          * so only kiq need set this field.
4550          */
4551         if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ)
4552                 mqd->cp_hqd_active = 1;
4553
4554         return 0;
4555 }
4556
4557 static int gfx_v8_0_mqd_commit(struct amdgpu_device *adev,
4558                         struct vi_mqd *mqd)
4559 {
4560         uint32_t mqd_reg;
4561         uint32_t *mqd_data;
4562
4563         /* HQD registers extend from mmCP_MQD_BASE_ADDR to mmCP_HQD_ERROR */
4564         mqd_data = &mqd->cp_mqd_base_addr_lo;
4565
4566         /* disable wptr polling */
4567         WREG32_FIELD(CP_PQ_WPTR_POLL_CNTL, EN, 0);
4568
4569         /* program all HQD registers */
4570         for (mqd_reg = mmCP_HQD_VMID; mqd_reg <= mmCP_HQD_EOP_CONTROL; mqd_reg++)
4571                 WREG32(mqd_reg, mqd_data[mqd_reg - mmCP_MQD_BASE_ADDR]);
4572
4573         /* Tonga errata: EOP RPTR/WPTR should be left unmodified.
4574          * This is safe since EOP RPTR==WPTR for any inactive HQD
4575          * on ASICs that do not support context-save.
4576          * EOP writes/reads can start anywhere in the ring.
4577          */
4578         if (adev->asic_type != CHIP_TONGA) {
4579                 WREG32(mmCP_HQD_EOP_RPTR, mqd->cp_hqd_eop_rptr);
4580                 WREG32(mmCP_HQD_EOP_WPTR, mqd->cp_hqd_eop_wptr);
4581                 WREG32(mmCP_HQD_EOP_WPTR_MEM, mqd->cp_hqd_eop_wptr_mem);
4582         }
4583
4584         for (mqd_reg = mmCP_HQD_EOP_EVENTS; mqd_reg <= mmCP_HQD_ERROR; mqd_reg++)
4585                 WREG32(mqd_reg, mqd_data[mqd_reg - mmCP_MQD_BASE_ADDR]);
4586
4587         /* activate the HQD */
4588         for (mqd_reg = mmCP_MQD_BASE_ADDR; mqd_reg <= mmCP_HQD_ACTIVE; mqd_reg++)
4589                 WREG32(mqd_reg, mqd_data[mqd_reg - mmCP_MQD_BASE_ADDR]);
4590
4591         return 0;
4592 }
4593
4594 static int gfx_v8_0_kiq_init_queue(struct amdgpu_ring *ring)
4595 {
4596         struct amdgpu_device *adev = ring->adev;
4597         struct vi_mqd *mqd = ring->mqd_ptr;
4598         int mqd_idx = AMDGPU_MAX_COMPUTE_RINGS;
4599
4600         gfx_v8_0_kiq_setting(ring);
4601
4602         if (amdgpu_in_reset(adev)) { /* for GPU_RESET case */
4603                 /* reset MQD to a clean status */
4604                 if (adev->gfx.mec.mqd_backup[mqd_idx])
4605                         memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct vi_mqd_allocation));
4606
4607                 /* reset ring buffer */
4608                 ring->wptr = 0;
4609                 amdgpu_ring_clear_ring(ring);
4610                 mutex_lock(&adev->srbm_mutex);
4611                 vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
4612                 gfx_v8_0_mqd_commit(adev, mqd);
4613                 vi_srbm_select(adev, 0, 0, 0, 0);
4614                 mutex_unlock(&adev->srbm_mutex);
4615         } else {
4616                 memset((void *)mqd, 0, sizeof(struct vi_mqd_allocation));
4617                 ((struct vi_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;
4618                 ((struct vi_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;
4619                 if (amdgpu_sriov_vf(adev) && adev->in_suspend)
4620                         amdgpu_ring_clear_ring(ring);
4621                 mutex_lock(&adev->srbm_mutex);
4622                 vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
4623                 gfx_v8_0_mqd_init(ring);
4624                 gfx_v8_0_mqd_commit(adev, mqd);
4625                 vi_srbm_select(adev, 0, 0, 0, 0);
4626                 mutex_unlock(&adev->srbm_mutex);
4627
4628                 if (adev->gfx.mec.mqd_backup[mqd_idx])
4629                         memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct vi_mqd_allocation));
4630         }
4631
4632         return 0;
4633 }
4634
4635 static int gfx_v8_0_kcq_init_queue(struct amdgpu_ring *ring)
4636 {
4637         struct amdgpu_device *adev = ring->adev;
4638         struct vi_mqd *mqd = ring->mqd_ptr;
4639         int mqd_idx = ring - &adev->gfx.compute_ring[0];
4640
4641         if (!amdgpu_in_reset(adev) && !adev->in_suspend) {
4642                 memset((void *)mqd, 0, sizeof(struct vi_mqd_allocation));
4643                 ((struct vi_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;
4644                 ((struct vi_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;
4645                 mutex_lock(&adev->srbm_mutex);
4646                 vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
4647                 gfx_v8_0_mqd_init(ring);
4648                 vi_srbm_select(adev, 0, 0, 0, 0);
4649                 mutex_unlock(&adev->srbm_mutex);
4650
4651                 if (adev->gfx.mec.mqd_backup[mqd_idx])
4652                         memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct vi_mqd_allocation));
4653         } else if (amdgpu_in_reset(adev)) { /* for GPU_RESET case */
4654                 /* reset MQD to a clean status */
4655                 if (adev->gfx.mec.mqd_backup[mqd_idx])
4656                         memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct vi_mqd_allocation));
4657                 /* reset ring buffer */
4658                 ring->wptr = 0;
4659                 amdgpu_ring_clear_ring(ring);
4660         } else {
4661                 amdgpu_ring_clear_ring(ring);
4662         }
4663         return 0;
4664 }
4665
4666 static void gfx_v8_0_set_mec_doorbell_range(struct amdgpu_device *adev)
4667 {
4668         if (adev->asic_type > CHIP_TONGA) {
4669                 WREG32(mmCP_MEC_DOORBELL_RANGE_LOWER, adev->doorbell_index.kiq << 2);
4670                 WREG32(mmCP_MEC_DOORBELL_RANGE_UPPER, adev->doorbell_index.mec_ring7 << 2);
4671         }
4672         /* enable doorbells */
4673         WREG32_FIELD(CP_PQ_STATUS, DOORBELL_ENABLE, 1);
4674 }
4675
4676 static int gfx_v8_0_kiq_resume(struct amdgpu_device *adev)
4677 {
4678         struct amdgpu_ring *ring;
4679         int r;
4680
4681         ring = &adev->gfx.kiq.ring;
4682
4683         r = amdgpu_bo_reserve(ring->mqd_obj, false);
4684         if (unlikely(r != 0))
4685                 return r;
4686
4687         r = amdgpu_bo_kmap(ring->mqd_obj, &ring->mqd_ptr);
4688         if (unlikely(r != 0))
4689                 return r;
4690
4691         gfx_v8_0_kiq_init_queue(ring);
4692         amdgpu_bo_kunmap(ring->mqd_obj);
4693         ring->mqd_ptr = NULL;
4694         amdgpu_bo_unreserve(ring->mqd_obj);
4695         ring->sched.ready = true;
4696         return 0;
4697 }
4698
4699 static int gfx_v8_0_kcq_resume(struct amdgpu_device *adev)
4700 {
4701         struct amdgpu_ring *ring = NULL;
4702         int r = 0, i;
4703
4704         gfx_v8_0_cp_compute_enable(adev, true);
4705
4706         for (i = 0; i < adev->gfx.num_compute_rings; i++) {
4707                 ring = &adev->gfx.compute_ring[i];
4708
4709                 r = amdgpu_bo_reserve(ring->mqd_obj, false);
4710                 if (unlikely(r != 0))
4711                         goto done;
4712                 r = amdgpu_bo_kmap(ring->mqd_obj, &ring->mqd_ptr);
4713                 if (!r) {
4714                         r = gfx_v8_0_kcq_init_queue(ring);
4715                         amdgpu_bo_kunmap(ring->mqd_obj);
4716                         ring->mqd_ptr = NULL;
4717                 }
4718                 amdgpu_bo_unreserve(ring->mqd_obj);
4719                 if (r)
4720                         goto done;
4721         }
4722
4723         gfx_v8_0_set_mec_doorbell_range(adev);
4724
4725         r = gfx_v8_0_kiq_kcq_enable(adev);
4726         if (r)
4727                 goto done;
4728
4729 done:
4730         return r;
4731 }
4732
4733 static int gfx_v8_0_cp_test_all_rings(struct amdgpu_device *adev)
4734 {
4735         int r, i;
4736         struct amdgpu_ring *ring;
4737
4738         /* collect all the ring_tests here, gfx, kiq, compute */
4739         ring = &adev->gfx.gfx_ring[0];
4740         r = amdgpu_ring_test_helper(ring);
4741         if (r)
4742                 return r;
4743
4744         ring = &adev->gfx.kiq.ring;
4745         r = amdgpu_ring_test_helper(ring);
4746         if (r)
4747                 return r;
4748
4749         for (i = 0; i < adev->gfx.num_compute_rings; i++) {
4750                 ring = &adev->gfx.compute_ring[i];
4751                 amdgpu_ring_test_helper(ring);
4752         }
4753
4754         return 0;
4755 }
4756
4757 static int gfx_v8_0_cp_resume(struct amdgpu_device *adev)
4758 {
4759         int r;
4760
4761         if (!(adev->flags & AMD_IS_APU))
4762                 gfx_v8_0_enable_gui_idle_interrupt(adev, false);
4763
4764         r = gfx_v8_0_kiq_resume(adev);
4765         if (r)
4766                 return r;
4767
4768         r = gfx_v8_0_cp_gfx_resume(adev);
4769         if (r)
4770                 return r;
4771
4772         r = gfx_v8_0_kcq_resume(adev);
4773         if (r)
4774                 return r;
4775
4776         r = gfx_v8_0_cp_test_all_rings(adev);
4777         if (r)
4778                 return r;
4779
4780         gfx_v8_0_enable_gui_idle_interrupt(adev, true);
4781
4782         return 0;
4783 }
4784
4785 static void gfx_v8_0_cp_enable(struct amdgpu_device *adev, bool enable)
4786 {
4787         gfx_v8_0_cp_gfx_enable(adev, enable);
4788         gfx_v8_0_cp_compute_enable(adev, enable);
4789 }
4790
4791 static int gfx_v8_0_hw_init(void *handle)
4792 {
4793         int r;
4794         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4795
4796         gfx_v8_0_init_golden_registers(adev);
4797         gfx_v8_0_constants_init(adev);
4798
4799         r = adev->gfx.rlc.funcs->resume(adev);
4800         if (r)
4801                 return r;
4802
4803         r = gfx_v8_0_cp_resume(adev);
4804
4805         return r;
4806 }
4807
4808 static int gfx_v8_0_kcq_disable(struct amdgpu_device *adev)
4809 {
4810         int r, i;
4811         struct amdgpu_ring *kiq_ring = &adev->gfx.kiq.ring;
4812
4813         r = amdgpu_ring_alloc(kiq_ring, 6 * adev->gfx.num_compute_rings);
4814         if (r)
4815                 DRM_ERROR("Failed to lock KIQ (%d).\n", r);
4816
4817         for (i = 0; i < adev->gfx.num_compute_rings; i++) {
4818                 struct amdgpu_ring *ring = &adev->gfx.compute_ring[i];
4819
4820                 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4));
4821                 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
4822                                                 PACKET3_UNMAP_QUEUES_ACTION(1) | /* RESET_QUEUES */
4823                                                 PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) |
4824                                                 PACKET3_UNMAP_QUEUES_ENGINE_SEL(0) |
4825                                                 PACKET3_UNMAP_QUEUES_NUM_QUEUES(1));
4826                 amdgpu_ring_write(kiq_ring, PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index));
4827                 amdgpu_ring_write(kiq_ring, 0);
4828                 amdgpu_ring_write(kiq_ring, 0);
4829                 amdgpu_ring_write(kiq_ring, 0);
4830         }
4831         r = amdgpu_ring_test_helper(kiq_ring);
4832         if (r)
4833                 DRM_ERROR("KCQ disable failed\n");
4834
4835         return r;
4836 }
4837
4838 static bool gfx_v8_0_is_idle(void *handle)
4839 {
4840         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4841
4842         if (REG_GET_FIELD(RREG32(mmGRBM_STATUS), GRBM_STATUS, GUI_ACTIVE)
4843                 || RREG32(mmGRBM_STATUS2) != 0x8)
4844                 return false;
4845         else
4846                 return true;
4847 }
4848
4849 static bool gfx_v8_0_rlc_is_idle(void *handle)
4850 {
4851         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4852
4853         if (RREG32(mmGRBM_STATUS2) != 0x8)
4854                 return false;
4855         else
4856                 return true;
4857 }
4858
4859 static int gfx_v8_0_wait_for_rlc_idle(void *handle)
4860 {
4861         unsigned int i;
4862         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4863
4864         for (i = 0; i < adev->usec_timeout; i++) {
4865                 if (gfx_v8_0_rlc_is_idle(handle))
4866                         return 0;
4867
4868                 udelay(1);
4869         }
4870         return -ETIMEDOUT;
4871 }
4872
4873 static int gfx_v8_0_wait_for_idle(void *handle)
4874 {
4875         unsigned int i;
4876         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4877
4878         for (i = 0; i < adev->usec_timeout; i++) {
4879                 if (gfx_v8_0_is_idle(handle))
4880                         return 0;
4881
4882                 udelay(1);
4883         }
4884         return -ETIMEDOUT;
4885 }
4886
4887 static int gfx_v8_0_hw_fini(void *handle)
4888 {
4889         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4890
4891         amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
4892         amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
4893
4894         amdgpu_irq_put(adev, &adev->gfx.cp_ecc_error_irq, 0);
4895
4896         amdgpu_irq_put(adev, &adev->gfx.sq_irq, 0);
4897
4898         /* disable KCQ to avoid CPC touch memory not valid anymore */
4899         gfx_v8_0_kcq_disable(adev);
4900
4901         if (amdgpu_sriov_vf(adev)) {
4902                 pr_debug("For SRIOV client, shouldn't do anything.\n");
4903                 return 0;
4904         }
4905         amdgpu_gfx_rlc_enter_safe_mode(adev);
4906         if (!gfx_v8_0_wait_for_idle(adev))
4907                 gfx_v8_0_cp_enable(adev, false);
4908         else
4909                 pr_err("cp is busy, skip halt cp\n");
4910         if (!gfx_v8_0_wait_for_rlc_idle(adev))
4911                 adev->gfx.rlc.funcs->stop(adev);
4912         else
4913                 pr_err("rlc is busy, skip halt rlc\n");
4914         amdgpu_gfx_rlc_exit_safe_mode(adev);
4915
4916         return 0;
4917 }
4918
4919 static int gfx_v8_0_suspend(void *handle)
4920 {
4921         return gfx_v8_0_hw_fini(handle);
4922 }
4923
4924 static int gfx_v8_0_resume(void *handle)
4925 {
4926         return gfx_v8_0_hw_init(handle);
4927 }
4928
4929 static bool gfx_v8_0_check_soft_reset(void *handle)
4930 {
4931         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4932         u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
4933         u32 tmp;
4934
4935         /* GRBM_STATUS */
4936         tmp = RREG32(mmGRBM_STATUS);
4937         if (tmp & (GRBM_STATUS__PA_BUSY_MASK | GRBM_STATUS__SC_BUSY_MASK |
4938                    GRBM_STATUS__BCI_BUSY_MASK | GRBM_STATUS__SX_BUSY_MASK |
4939                    GRBM_STATUS__TA_BUSY_MASK | GRBM_STATUS__VGT_BUSY_MASK |
4940                    GRBM_STATUS__DB_BUSY_MASK | GRBM_STATUS__CB_BUSY_MASK |
4941                    GRBM_STATUS__GDS_BUSY_MASK | GRBM_STATUS__SPI_BUSY_MASK |
4942                    GRBM_STATUS__IA_BUSY_MASK | GRBM_STATUS__IA_BUSY_NO_DMA_MASK |
4943                    GRBM_STATUS__CP_BUSY_MASK | GRBM_STATUS__CP_COHERENCY_BUSY_MASK)) {
4944                 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4945                                                 GRBM_SOFT_RESET, SOFT_RESET_CP, 1);
4946                 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4947                                                 GRBM_SOFT_RESET, SOFT_RESET_GFX, 1);
4948                 srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset,
4949                                                 SRBM_SOFT_RESET, SOFT_RESET_GRBM, 1);
4950         }
4951
4952         /* GRBM_STATUS2 */
4953         tmp = RREG32(mmGRBM_STATUS2);
4954         if (REG_GET_FIELD(tmp, GRBM_STATUS2, RLC_BUSY))
4955                 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4956                                                 GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
4957
4958         if (REG_GET_FIELD(tmp, GRBM_STATUS2, CPF_BUSY) ||
4959             REG_GET_FIELD(tmp, GRBM_STATUS2, CPC_BUSY) ||
4960             REG_GET_FIELD(tmp, GRBM_STATUS2, CPG_BUSY)) {
4961                 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
4962                                                 SOFT_RESET_CPF, 1);
4963                 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
4964                                                 SOFT_RESET_CPC, 1);
4965                 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
4966                                                 SOFT_RESET_CPG, 1);
4967                 srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET,
4968                                                 SOFT_RESET_GRBM, 1);
4969         }
4970
4971         /* SRBM_STATUS */
4972         tmp = RREG32(mmSRBM_STATUS);
4973         if (REG_GET_FIELD(tmp, SRBM_STATUS, GRBM_RQ_PENDING))
4974                 srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset,
4975                                                 SRBM_SOFT_RESET, SOFT_RESET_GRBM, 1);
4976         if (REG_GET_FIELD(tmp, SRBM_STATUS, SEM_BUSY))
4977                 srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset,
4978                                                 SRBM_SOFT_RESET, SOFT_RESET_SEM, 1);
4979
4980         if (grbm_soft_reset || srbm_soft_reset) {
4981                 adev->gfx.grbm_soft_reset = grbm_soft_reset;
4982                 adev->gfx.srbm_soft_reset = srbm_soft_reset;
4983                 return true;
4984         } else {
4985                 adev->gfx.grbm_soft_reset = 0;
4986                 adev->gfx.srbm_soft_reset = 0;
4987                 return false;
4988         }
4989 }
4990
4991 static int gfx_v8_0_pre_soft_reset(void *handle)
4992 {
4993         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4994         u32 grbm_soft_reset = 0;
4995
4996         if ((!adev->gfx.grbm_soft_reset) &&
4997             (!adev->gfx.srbm_soft_reset))
4998                 return 0;
4999
5000         grbm_soft_reset = adev->gfx.grbm_soft_reset;
5001
5002         /* stop the rlc */
5003         adev->gfx.rlc.funcs->stop(adev);
5004
5005         if (REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CP) ||
5006             REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_GFX))
5007                 /* Disable GFX parsing/prefetching */
5008                 gfx_v8_0_cp_gfx_enable(adev, false);
5009
5010         if (REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CP) ||
5011             REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CPF) ||
5012             REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CPC) ||
5013             REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CPG)) {
5014                 int i;
5015
5016                 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
5017                         struct amdgpu_ring *ring = &adev->gfx.compute_ring[i];
5018
5019                         mutex_lock(&adev->srbm_mutex);
5020                         vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
5021                         gfx_v8_0_deactivate_hqd(adev, 2);
5022                         vi_srbm_select(adev, 0, 0, 0, 0);
5023                         mutex_unlock(&adev->srbm_mutex);
5024                 }
5025                 /* Disable MEC parsing/prefetching */
5026                 gfx_v8_0_cp_compute_enable(adev, false);
5027         }
5028
5029         return 0;
5030 }
5031
5032 static int gfx_v8_0_soft_reset(void *handle)
5033 {
5034         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5035         u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
5036         u32 tmp;
5037
5038         if ((!adev->gfx.grbm_soft_reset) &&
5039             (!adev->gfx.srbm_soft_reset))
5040                 return 0;
5041
5042         grbm_soft_reset = adev->gfx.grbm_soft_reset;
5043         srbm_soft_reset = adev->gfx.srbm_soft_reset;
5044
5045         if (grbm_soft_reset || srbm_soft_reset) {
5046                 tmp = RREG32(mmGMCON_DEBUG);
5047                 tmp = REG_SET_FIELD(tmp, GMCON_DEBUG, GFX_STALL, 1);
5048                 tmp = REG_SET_FIELD(tmp, GMCON_DEBUG, GFX_CLEAR, 1);
5049                 WREG32(mmGMCON_DEBUG, tmp);
5050                 udelay(50);
5051         }
5052
5053         if (grbm_soft_reset) {
5054                 tmp = RREG32(mmGRBM_SOFT_RESET);
5055                 tmp |= grbm_soft_reset;
5056                 dev_info(adev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
5057                 WREG32(mmGRBM_SOFT_RESET, tmp);
5058                 tmp = RREG32(mmGRBM_SOFT_RESET);
5059
5060                 udelay(50);
5061
5062                 tmp &= ~grbm_soft_reset;
5063                 WREG32(mmGRBM_SOFT_RESET, tmp);
5064                 tmp = RREG32(mmGRBM_SOFT_RESET);
5065         }
5066
5067         if (srbm_soft_reset) {
5068                 tmp = RREG32(mmSRBM_SOFT_RESET);
5069                 tmp |= srbm_soft_reset;
5070                 dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
5071                 WREG32(mmSRBM_SOFT_RESET, tmp);
5072                 tmp = RREG32(mmSRBM_SOFT_RESET);
5073
5074                 udelay(50);
5075
5076                 tmp &= ~srbm_soft_reset;
5077                 WREG32(mmSRBM_SOFT_RESET, tmp);
5078                 tmp = RREG32(mmSRBM_SOFT_RESET);
5079         }
5080
5081         if (grbm_soft_reset || srbm_soft_reset) {
5082                 tmp = RREG32(mmGMCON_DEBUG);
5083                 tmp = REG_SET_FIELD(tmp, GMCON_DEBUG, GFX_STALL, 0);
5084                 tmp = REG_SET_FIELD(tmp, GMCON_DEBUG, GFX_CLEAR, 0);
5085                 WREG32(mmGMCON_DEBUG, tmp);
5086         }
5087
5088         /* Wait a little for things to settle down */
5089         udelay(50);
5090
5091         return 0;
5092 }
5093
5094 static int gfx_v8_0_post_soft_reset(void *handle)
5095 {
5096         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5097         u32 grbm_soft_reset = 0;
5098
5099         if ((!adev->gfx.grbm_soft_reset) &&
5100             (!adev->gfx.srbm_soft_reset))
5101                 return 0;
5102
5103         grbm_soft_reset = adev->gfx.grbm_soft_reset;
5104
5105         if (REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CP) ||
5106             REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CPF) ||
5107             REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CPC) ||
5108             REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CPG)) {
5109                 int i;
5110
5111                 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
5112                         struct amdgpu_ring *ring = &adev->gfx.compute_ring[i];
5113
5114                         mutex_lock(&adev->srbm_mutex);
5115                         vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
5116                         gfx_v8_0_deactivate_hqd(adev, 2);
5117                         vi_srbm_select(adev, 0, 0, 0, 0);
5118                         mutex_unlock(&adev->srbm_mutex);
5119                 }
5120                 gfx_v8_0_kiq_resume(adev);
5121                 gfx_v8_0_kcq_resume(adev);
5122         }
5123
5124         if (REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CP) ||
5125             REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_GFX))
5126                 gfx_v8_0_cp_gfx_resume(adev);
5127
5128         gfx_v8_0_cp_test_all_rings(adev);
5129
5130         adev->gfx.rlc.funcs->start(adev);
5131
5132         return 0;
5133 }
5134
5135 /**
5136  * gfx_v8_0_get_gpu_clock_counter - return GPU clock counter snapshot
5137  *
5138  * @adev: amdgpu_device pointer
5139  *
5140  * Fetches a GPU clock counter snapshot.
5141  * Returns the 64 bit clock counter snapshot.
5142  */
5143 static uint64_t gfx_v8_0_get_gpu_clock_counter(struct amdgpu_device *adev)
5144 {
5145         uint64_t clock;
5146
5147         mutex_lock(&adev->gfx.gpu_clock_mutex);
5148         WREG32(mmRLC_CAPTURE_GPU_CLOCK_COUNT, 1);
5149         clock = (uint64_t)RREG32(mmRLC_GPU_CLOCK_COUNT_LSB) |
5150                 ((uint64_t)RREG32(mmRLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
5151         mutex_unlock(&adev->gfx.gpu_clock_mutex);
5152         return clock;
5153 }
5154
5155 static void gfx_v8_0_ring_emit_gds_switch(struct amdgpu_ring *ring,
5156                                           uint32_t vmid,
5157                                           uint32_t gds_base, uint32_t gds_size,
5158                                           uint32_t gws_base, uint32_t gws_size,
5159                                           uint32_t oa_base, uint32_t oa_size)
5160 {
5161         /* GDS Base */
5162         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5163         amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5164                                 WRITE_DATA_DST_SEL(0)));
5165         amdgpu_ring_write(ring, amdgpu_gds_reg_offset[vmid].mem_base);
5166         amdgpu_ring_write(ring, 0);
5167         amdgpu_ring_write(ring, gds_base);
5168
5169         /* GDS Size */
5170         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5171         amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5172                                 WRITE_DATA_DST_SEL(0)));
5173         amdgpu_ring_write(ring, amdgpu_gds_reg_offset[vmid].mem_size);
5174         amdgpu_ring_write(ring, 0);
5175         amdgpu_ring_write(ring, gds_size);
5176
5177         /* GWS */
5178         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5179         amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5180                                 WRITE_DATA_DST_SEL(0)));
5181         amdgpu_ring_write(ring, amdgpu_gds_reg_offset[vmid].gws);
5182         amdgpu_ring_write(ring, 0);
5183         amdgpu_ring_write(ring, gws_size << GDS_GWS_VMID0__SIZE__SHIFT | gws_base);
5184
5185         /* OA */
5186         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5187         amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5188                                 WRITE_DATA_DST_SEL(0)));
5189         amdgpu_ring_write(ring, amdgpu_gds_reg_offset[vmid].oa);
5190         amdgpu_ring_write(ring, 0);
5191         amdgpu_ring_write(ring, (1 << (oa_size + oa_base)) - (1 << oa_base));
5192 }
5193
5194 static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t address)
5195 {
5196         WREG32(mmSQ_IND_INDEX,
5197                 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
5198                 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
5199                 (address << SQ_IND_INDEX__INDEX__SHIFT) |
5200                 (SQ_IND_INDEX__FORCE_READ_MASK));
5201         return RREG32(mmSQ_IND_DATA);
5202 }
5203
5204 static void wave_read_regs(struct amdgpu_device *adev, uint32_t simd,
5205                            uint32_t wave, uint32_t thread,
5206                            uint32_t regno, uint32_t num, uint32_t *out)
5207 {
5208         WREG32(mmSQ_IND_INDEX,
5209                 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
5210                 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
5211                 (regno << SQ_IND_INDEX__INDEX__SHIFT) |
5212                 (thread << SQ_IND_INDEX__THREAD_ID__SHIFT) |
5213                 (SQ_IND_INDEX__FORCE_READ_MASK) |
5214                 (SQ_IND_INDEX__AUTO_INCR_MASK));
5215         while (num--)
5216                 *(out++) = RREG32(mmSQ_IND_DATA);
5217 }
5218
5219 static void gfx_v8_0_read_wave_data(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields)
5220 {
5221         /* type 0 wave data */
5222         dst[(*no_fields)++] = 0;
5223         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_STATUS);
5224         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_LO);
5225         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_HI);
5226         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_LO);
5227         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_HI);
5228         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_HW_ID);
5229         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW0);
5230         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW1);
5231         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_GPR_ALLOC);
5232         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_LDS_ALLOC);
5233         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TRAPSTS);
5234         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_STS);
5235         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TBA_LO);
5236         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TBA_HI);
5237         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TMA_LO);
5238         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TMA_HI);
5239         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_DBG0);
5240         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_M0);
5241         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_MODE);
5242 }
5243
5244 static void gfx_v8_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t simd,
5245                                      uint32_t wave, uint32_t start,
5246                                      uint32_t size, uint32_t *dst)
5247 {
5248         wave_read_regs(
5249                 adev, simd, wave, 0,
5250                 start + SQIND_WAVE_SGPRS_OFFSET, size, dst);
5251 }
5252
5253
5254 static const struct amdgpu_gfx_funcs gfx_v8_0_gfx_funcs = {
5255         .get_gpu_clock_counter = &gfx_v8_0_get_gpu_clock_counter,
5256         .select_se_sh = &gfx_v8_0_select_se_sh,
5257         .read_wave_data = &gfx_v8_0_read_wave_data,
5258         .read_wave_sgprs = &gfx_v8_0_read_wave_sgprs,
5259         .select_me_pipe_q = &gfx_v8_0_select_me_pipe_q
5260 };
5261
5262 static int gfx_v8_0_early_init(void *handle)
5263 {
5264         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5265
5266         adev->gfx.num_gfx_rings = GFX8_NUM_GFX_RINGS;
5267         adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev),
5268                                           AMDGPU_MAX_COMPUTE_RINGS);
5269         adev->gfx.funcs = &gfx_v8_0_gfx_funcs;
5270         gfx_v8_0_set_ring_funcs(adev);
5271         gfx_v8_0_set_irq_funcs(adev);
5272         gfx_v8_0_set_gds_init(adev);
5273         gfx_v8_0_set_rlc_funcs(adev);
5274
5275         return 0;
5276 }
5277
5278 static int gfx_v8_0_late_init(void *handle)
5279 {
5280         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5281         int r;
5282
5283         r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0);
5284         if (r)
5285                 return r;
5286
5287         r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0);
5288         if (r)
5289                 return r;
5290
5291         /* requires IBs so do in late init after IB pool is initialized */
5292         r = gfx_v8_0_do_edc_gpr_workarounds(adev);
5293         if (r)
5294                 return r;
5295
5296         r = amdgpu_irq_get(adev, &adev->gfx.cp_ecc_error_irq, 0);
5297         if (r) {
5298                 DRM_ERROR("amdgpu_irq_get() failed to get IRQ for EDC, r: %d.\n", r);
5299                 return r;
5300         }
5301
5302         r = amdgpu_irq_get(adev, &adev->gfx.sq_irq, 0);
5303         if (r) {
5304                 DRM_ERROR(
5305                         "amdgpu_irq_get() failed to get IRQ for SQ, r: %d.\n",
5306                         r);
5307                 return r;
5308         }
5309
5310         return 0;
5311 }
5312
5313 static void gfx_v8_0_enable_gfx_static_mg_power_gating(struct amdgpu_device *adev,
5314                                                        bool enable)
5315 {
5316         if ((adev->asic_type == CHIP_POLARIS11) ||
5317             (adev->asic_type == CHIP_POLARIS12) ||
5318             (adev->asic_type == CHIP_VEGAM))
5319                 /* Send msg to SMU via Powerplay */
5320                 amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, enable);
5321
5322         WREG32_FIELD(RLC_PG_CNTL, STATIC_PER_CU_PG_ENABLE, enable ? 1 : 0);
5323 }
5324
5325 static void gfx_v8_0_enable_gfx_dynamic_mg_power_gating(struct amdgpu_device *adev,
5326                                                         bool enable)
5327 {
5328         WREG32_FIELD(RLC_PG_CNTL, DYN_PER_CU_PG_ENABLE, enable ? 1 : 0);
5329 }
5330
5331 static void polaris11_enable_gfx_quick_mg_power_gating(struct amdgpu_device *adev,
5332                 bool enable)
5333 {
5334         WREG32_FIELD(RLC_PG_CNTL, QUICK_PG_ENABLE, enable ? 1 : 0);
5335 }
5336
5337 static void cz_enable_gfx_cg_power_gating(struct amdgpu_device *adev,
5338                                           bool enable)
5339 {
5340         WREG32_FIELD(RLC_PG_CNTL, GFX_POWER_GATING_ENABLE, enable ? 1 : 0);
5341 }
5342
5343 static void cz_enable_gfx_pipeline_power_gating(struct amdgpu_device *adev,
5344                                                 bool enable)
5345 {
5346         WREG32_FIELD(RLC_PG_CNTL, GFX_PIPELINE_PG_ENABLE, enable ? 1 : 0);
5347
5348         /* Read any GFX register to wake up GFX. */
5349         if (!enable)
5350                 RREG32(mmDB_RENDER_CONTROL);
5351 }
5352
5353 static void cz_update_gfx_cg_power_gating(struct amdgpu_device *adev,
5354                                           bool enable)
5355 {
5356         if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_PG) && enable) {
5357                 cz_enable_gfx_cg_power_gating(adev, true);
5358                 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE)
5359                         cz_enable_gfx_pipeline_power_gating(adev, true);
5360         } else {
5361                 cz_enable_gfx_cg_power_gating(adev, false);
5362                 cz_enable_gfx_pipeline_power_gating(adev, false);
5363         }
5364 }
5365
5366 static int gfx_v8_0_set_powergating_state(void *handle,
5367                                           enum amd_powergating_state state)
5368 {
5369         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5370         bool enable = (state == AMD_PG_STATE_GATE);
5371
5372         if (amdgpu_sriov_vf(adev))
5373                 return 0;
5374
5375         if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_SMG |
5376                                 AMD_PG_SUPPORT_RLC_SMU_HS |
5377                                 AMD_PG_SUPPORT_CP |
5378                                 AMD_PG_SUPPORT_GFX_DMG))
5379                 amdgpu_gfx_rlc_enter_safe_mode(adev);
5380         switch (adev->asic_type) {
5381         case CHIP_CARRIZO:
5382         case CHIP_STONEY:
5383
5384                 if (adev->pg_flags & AMD_PG_SUPPORT_RLC_SMU_HS) {
5385                         cz_enable_sck_slow_down_on_power_up(adev, true);
5386                         cz_enable_sck_slow_down_on_power_down(adev, true);
5387                 } else {
5388                         cz_enable_sck_slow_down_on_power_up(adev, false);
5389                         cz_enable_sck_slow_down_on_power_down(adev, false);
5390                 }
5391                 if (adev->pg_flags & AMD_PG_SUPPORT_CP)
5392                         cz_enable_cp_power_gating(adev, true);
5393                 else
5394                         cz_enable_cp_power_gating(adev, false);
5395
5396                 cz_update_gfx_cg_power_gating(adev, enable);
5397
5398                 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG) && enable)
5399                         gfx_v8_0_enable_gfx_static_mg_power_gating(adev, true);
5400                 else
5401                         gfx_v8_0_enable_gfx_static_mg_power_gating(adev, false);
5402
5403                 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG) && enable)
5404                         gfx_v8_0_enable_gfx_dynamic_mg_power_gating(adev, true);
5405                 else
5406                         gfx_v8_0_enable_gfx_dynamic_mg_power_gating(adev, false);
5407                 break;
5408         case CHIP_POLARIS11:
5409         case CHIP_POLARIS12:
5410         case CHIP_VEGAM:
5411                 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG) && enable)
5412                         gfx_v8_0_enable_gfx_static_mg_power_gating(adev, true);
5413                 else
5414                         gfx_v8_0_enable_gfx_static_mg_power_gating(adev, false);
5415
5416                 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG) && enable)
5417                         gfx_v8_0_enable_gfx_dynamic_mg_power_gating(adev, true);
5418                 else
5419                         gfx_v8_0_enable_gfx_dynamic_mg_power_gating(adev, false);
5420
5421                 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_QUICK_MG) && enable)
5422                         polaris11_enable_gfx_quick_mg_power_gating(adev, true);
5423                 else
5424                         polaris11_enable_gfx_quick_mg_power_gating(adev, false);
5425                 break;
5426         default:
5427                 break;
5428         }
5429         if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_SMG |
5430                                 AMD_PG_SUPPORT_RLC_SMU_HS |
5431                                 AMD_PG_SUPPORT_CP |
5432                                 AMD_PG_SUPPORT_GFX_DMG))
5433                 amdgpu_gfx_rlc_exit_safe_mode(adev);
5434         return 0;
5435 }
5436
5437 static void gfx_v8_0_get_clockgating_state(void *handle, u64 *flags)
5438 {
5439         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5440         int data;
5441
5442         if (amdgpu_sriov_vf(adev))
5443                 *flags = 0;
5444
5445         /* AMD_CG_SUPPORT_GFX_MGCG */
5446         data = RREG32(mmRLC_CGTT_MGCG_OVERRIDE);
5447         if (!(data & RLC_CGTT_MGCG_OVERRIDE__CPF_MASK))
5448                 *flags |= AMD_CG_SUPPORT_GFX_MGCG;
5449
5450         /* AMD_CG_SUPPORT_GFX_CGLG */
5451         data = RREG32(mmRLC_CGCG_CGLS_CTRL);
5452         if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK)
5453                 *flags |= AMD_CG_SUPPORT_GFX_CGCG;
5454
5455         /* AMD_CG_SUPPORT_GFX_CGLS */
5456         if (data & RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK)
5457                 *flags |= AMD_CG_SUPPORT_GFX_CGLS;
5458
5459         /* AMD_CG_SUPPORT_GFX_CGTS */
5460         data = RREG32(mmCGTS_SM_CTRL_REG);
5461         if (!(data & CGTS_SM_CTRL_REG__OVERRIDE_MASK))
5462                 *flags |= AMD_CG_SUPPORT_GFX_CGTS;
5463
5464         /* AMD_CG_SUPPORT_GFX_CGTS_LS */
5465         if (!(data & CGTS_SM_CTRL_REG__LS_OVERRIDE_MASK))
5466                 *flags |= AMD_CG_SUPPORT_GFX_CGTS_LS;
5467
5468         /* AMD_CG_SUPPORT_GFX_RLC_LS */
5469         data = RREG32(mmRLC_MEM_SLP_CNTL);
5470         if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK)
5471                 *flags |= AMD_CG_SUPPORT_GFX_RLC_LS | AMD_CG_SUPPORT_GFX_MGLS;
5472
5473         /* AMD_CG_SUPPORT_GFX_CP_LS */
5474         data = RREG32(mmCP_MEM_SLP_CNTL);
5475         if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK)
5476                 *flags |= AMD_CG_SUPPORT_GFX_CP_LS | AMD_CG_SUPPORT_GFX_MGLS;
5477 }
5478
5479 static void gfx_v8_0_send_serdes_cmd(struct amdgpu_device *adev,
5480                                      uint32_t reg_addr, uint32_t cmd)
5481 {
5482         uint32_t data;
5483
5484         gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
5485
5486         WREG32(mmRLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5487         WREG32(mmRLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5488
5489         data = RREG32(mmRLC_SERDES_WR_CTRL);
5490         if (adev->asic_type == CHIP_STONEY)
5491                 data &= ~(RLC_SERDES_WR_CTRL__WRITE_COMMAND_MASK |
5492                           RLC_SERDES_WR_CTRL__READ_COMMAND_MASK |
5493                           RLC_SERDES_WR_CTRL__P1_SELECT_MASK |
5494                           RLC_SERDES_WR_CTRL__P2_SELECT_MASK |
5495                           RLC_SERDES_WR_CTRL__RDDATA_RESET_MASK |
5496                           RLC_SERDES_WR_CTRL__POWER_DOWN_MASK |
5497                           RLC_SERDES_WR_CTRL__POWER_UP_MASK |
5498                           RLC_SERDES_WR_CTRL__SHORT_FORMAT_MASK |
5499                           RLC_SERDES_WR_CTRL__SRBM_OVERRIDE_MASK);
5500         else
5501                 data &= ~(RLC_SERDES_WR_CTRL__WRITE_COMMAND_MASK |
5502                           RLC_SERDES_WR_CTRL__READ_COMMAND_MASK |
5503                           RLC_SERDES_WR_CTRL__P1_SELECT_MASK |
5504                           RLC_SERDES_WR_CTRL__P2_SELECT_MASK |
5505                           RLC_SERDES_WR_CTRL__RDDATA_RESET_MASK |
5506                           RLC_SERDES_WR_CTRL__POWER_DOWN_MASK |
5507                           RLC_SERDES_WR_CTRL__POWER_UP_MASK |
5508                           RLC_SERDES_WR_CTRL__SHORT_FORMAT_MASK |
5509                           RLC_SERDES_WR_CTRL__BPM_DATA_MASK |
5510                           RLC_SERDES_WR_CTRL__REG_ADDR_MASK |
5511                           RLC_SERDES_WR_CTRL__SRBM_OVERRIDE_MASK);
5512         data |= (RLC_SERDES_WR_CTRL__RSVD_BPM_ADDR_MASK |
5513                  (cmd << RLC_SERDES_WR_CTRL__BPM_DATA__SHIFT) |
5514                  (reg_addr << RLC_SERDES_WR_CTRL__REG_ADDR__SHIFT) |
5515                  (0xff << RLC_SERDES_WR_CTRL__BPM_ADDR__SHIFT));
5516
5517         WREG32(mmRLC_SERDES_WR_CTRL, data);
5518 }
5519
5520 #define MSG_ENTER_RLC_SAFE_MODE     1
5521 #define MSG_EXIT_RLC_SAFE_MODE      0
5522 #define RLC_GPR_REG2__REQ_MASK 0x00000001
5523 #define RLC_GPR_REG2__REQ__SHIFT 0
5524 #define RLC_GPR_REG2__MESSAGE__SHIFT 0x00000001
5525 #define RLC_GPR_REG2__MESSAGE_MASK 0x0000001e
5526
5527 static bool gfx_v8_0_is_rlc_enabled(struct amdgpu_device *adev)
5528 {
5529         uint32_t rlc_setting;
5530
5531         rlc_setting = RREG32(mmRLC_CNTL);
5532         if (!(rlc_setting & RLC_CNTL__RLC_ENABLE_F32_MASK))
5533                 return false;
5534
5535         return true;
5536 }
5537
5538 static void gfx_v8_0_set_safe_mode(struct amdgpu_device *adev)
5539 {
5540         uint32_t data;
5541         unsigned i;
5542         data = RREG32(mmRLC_CNTL);
5543         data |= RLC_SAFE_MODE__CMD_MASK;
5544         data &= ~RLC_SAFE_MODE__MESSAGE_MASK;
5545         data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT);
5546         WREG32(mmRLC_SAFE_MODE, data);
5547
5548         /* wait for RLC_SAFE_MODE */
5549         for (i = 0; i < adev->usec_timeout; i++) {
5550                 if ((RREG32(mmRLC_GPM_STAT) &
5551                      (RLC_GPM_STAT__GFX_CLOCK_STATUS_MASK |
5552                       RLC_GPM_STAT__GFX_POWER_STATUS_MASK)) ==
5553                     (RLC_GPM_STAT__GFX_CLOCK_STATUS_MASK |
5554                      RLC_GPM_STAT__GFX_POWER_STATUS_MASK))
5555                         break;
5556                 udelay(1);
5557         }
5558         for (i = 0; i < adev->usec_timeout; i++) {
5559                 if (!REG_GET_FIELD(RREG32(mmRLC_SAFE_MODE), RLC_SAFE_MODE, CMD))
5560                         break;
5561                 udelay(1);
5562         }
5563 }
5564
5565 static void gfx_v8_0_unset_safe_mode(struct amdgpu_device *adev)
5566 {
5567         uint32_t data;
5568         unsigned i;
5569
5570         data = RREG32(mmRLC_CNTL);
5571         data |= RLC_SAFE_MODE__CMD_MASK;
5572         data &= ~RLC_SAFE_MODE__MESSAGE_MASK;
5573         WREG32(mmRLC_SAFE_MODE, data);
5574
5575         for (i = 0; i < adev->usec_timeout; i++) {
5576                 if (!REG_GET_FIELD(RREG32(mmRLC_SAFE_MODE), RLC_SAFE_MODE, CMD))
5577                         break;
5578                 udelay(1);
5579         }
5580 }
5581
5582 static void gfx_v8_0_update_spm_vmid(struct amdgpu_device *adev, unsigned vmid)
5583 {
5584         u32 data;
5585
5586         amdgpu_gfx_off_ctrl(adev, false);
5587
5588         if (amdgpu_sriov_is_pp_one_vf(adev))
5589                 data = RREG32_NO_KIQ(mmRLC_SPM_VMID);
5590         else
5591                 data = RREG32(mmRLC_SPM_VMID);
5592
5593         data &= ~RLC_SPM_VMID__RLC_SPM_VMID_MASK;
5594         data |= (vmid & RLC_SPM_VMID__RLC_SPM_VMID_MASK) << RLC_SPM_VMID__RLC_SPM_VMID__SHIFT;
5595
5596         if (amdgpu_sriov_is_pp_one_vf(adev))
5597                 WREG32_NO_KIQ(mmRLC_SPM_VMID, data);
5598         else
5599                 WREG32(mmRLC_SPM_VMID, data);
5600
5601         amdgpu_gfx_off_ctrl(adev, true);
5602 }
5603
5604 static const struct amdgpu_rlc_funcs iceland_rlc_funcs = {
5605         .is_rlc_enabled = gfx_v8_0_is_rlc_enabled,
5606         .set_safe_mode = gfx_v8_0_set_safe_mode,
5607         .unset_safe_mode = gfx_v8_0_unset_safe_mode,
5608         .init = gfx_v8_0_rlc_init,
5609         .get_csb_size = gfx_v8_0_get_csb_size,
5610         .get_csb_buffer = gfx_v8_0_get_csb_buffer,
5611         .get_cp_table_num = gfx_v8_0_cp_jump_table_num,
5612         .resume = gfx_v8_0_rlc_resume,
5613         .stop = gfx_v8_0_rlc_stop,
5614         .reset = gfx_v8_0_rlc_reset,
5615         .start = gfx_v8_0_rlc_start,
5616         .update_spm_vmid = gfx_v8_0_update_spm_vmid
5617 };
5618
5619 static void gfx_v8_0_update_medium_grain_clock_gating(struct amdgpu_device *adev,
5620                                                       bool enable)
5621 {
5622         uint32_t temp, data;
5623
5624         amdgpu_gfx_rlc_enter_safe_mode(adev);
5625
5626         /* It is disabled by HW by default */
5627         if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG)) {
5628                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) {
5629                         if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS)
5630                                 /* 1 - RLC memory Light sleep */
5631                                 WREG32_FIELD(RLC_MEM_SLP_CNTL, RLC_MEM_LS_EN, 1);
5632
5633                         if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS)
5634                                 WREG32_FIELD(CP_MEM_SLP_CNTL, CP_MEM_LS_EN, 1);
5635                 }
5636
5637                 /* 3 - RLC_CGTT_MGCG_OVERRIDE */
5638                 temp = data = RREG32(mmRLC_CGTT_MGCG_OVERRIDE);
5639                 if (adev->flags & AMD_IS_APU)
5640                         data &= ~(RLC_CGTT_MGCG_OVERRIDE__CPF_MASK |
5641                                   RLC_CGTT_MGCG_OVERRIDE__RLC_MASK |
5642                                   RLC_CGTT_MGCG_OVERRIDE__MGCG_MASK);
5643                 else
5644                         data &= ~(RLC_CGTT_MGCG_OVERRIDE__CPF_MASK |
5645                                   RLC_CGTT_MGCG_OVERRIDE__RLC_MASK |
5646                                   RLC_CGTT_MGCG_OVERRIDE__MGCG_MASK |
5647                                   RLC_CGTT_MGCG_OVERRIDE__GRBM_MASK);
5648
5649                 if (temp != data)
5650                         WREG32(mmRLC_CGTT_MGCG_OVERRIDE, data);
5651
5652                 /* 4 - wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */
5653                 gfx_v8_0_wait_for_rlc_serdes(adev);
5654
5655                 /* 5 - clear mgcg override */
5656                 gfx_v8_0_send_serdes_cmd(adev, BPM_REG_MGCG_OVERRIDE, CLE_BPM_SERDES_CMD);
5657
5658                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGTS) {
5659                         /* 6 - Enable CGTS(Tree Shade) MGCG /MGLS */
5660                         temp = data = RREG32(mmCGTS_SM_CTRL_REG);
5661                         data &= ~(CGTS_SM_CTRL_REG__SM_MODE_MASK);
5662                         data |= (0x2 << CGTS_SM_CTRL_REG__SM_MODE__SHIFT);
5663                         data |= CGTS_SM_CTRL_REG__SM_MODE_ENABLE_MASK;
5664                         data &= ~CGTS_SM_CTRL_REG__OVERRIDE_MASK;
5665                         if ((adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) &&
5666                             (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGTS_LS))
5667                                 data &= ~CGTS_SM_CTRL_REG__LS_OVERRIDE_MASK;
5668                         data |= CGTS_SM_CTRL_REG__ON_MONITOR_ADD_EN_MASK;
5669                         data |= (0x96 << CGTS_SM_CTRL_REG__ON_MONITOR_ADD__SHIFT);
5670                         if (temp != data)
5671                                 WREG32(mmCGTS_SM_CTRL_REG, data);
5672                 }
5673                 udelay(50);
5674
5675                 /* 7 - wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */
5676                 gfx_v8_0_wait_for_rlc_serdes(adev);
5677         } else {
5678                 /* 1 - MGCG_OVERRIDE[0] for CP and MGCG_OVERRIDE[1] for RLC */
5679                 temp = data = RREG32(mmRLC_CGTT_MGCG_OVERRIDE);
5680                 data |= (RLC_CGTT_MGCG_OVERRIDE__CPF_MASK |
5681                                 RLC_CGTT_MGCG_OVERRIDE__RLC_MASK |
5682                                 RLC_CGTT_MGCG_OVERRIDE__MGCG_MASK |
5683                                 RLC_CGTT_MGCG_OVERRIDE__GRBM_MASK);
5684                 if (temp != data)
5685                         WREG32(mmRLC_CGTT_MGCG_OVERRIDE, data);
5686
5687                 /* 2 - disable MGLS in RLC */
5688                 data = RREG32(mmRLC_MEM_SLP_CNTL);
5689                 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) {
5690                         data &= ~RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK;
5691                         WREG32(mmRLC_MEM_SLP_CNTL, data);
5692                 }
5693
5694                 /* 3 - disable MGLS in CP */
5695                 data = RREG32(mmCP_MEM_SLP_CNTL);
5696                 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) {
5697                         data &= ~CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK;
5698                         WREG32(mmCP_MEM_SLP_CNTL, data);
5699                 }
5700
5701                 /* 4 - Disable CGTS(Tree Shade) MGCG and MGLS */
5702                 temp = data = RREG32(mmCGTS_SM_CTRL_REG);
5703                 data |= (CGTS_SM_CTRL_REG__OVERRIDE_MASK |
5704                                 CGTS_SM_CTRL_REG__LS_OVERRIDE_MASK);
5705                 if (temp != data)
5706                         WREG32(mmCGTS_SM_CTRL_REG, data);
5707
5708                 /* 5 - wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */
5709                 gfx_v8_0_wait_for_rlc_serdes(adev);
5710
5711                 /* 6 - set mgcg override */
5712                 gfx_v8_0_send_serdes_cmd(adev, BPM_REG_MGCG_OVERRIDE, SET_BPM_SERDES_CMD);
5713
5714                 udelay(50);
5715
5716                 /* 7- wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */
5717                 gfx_v8_0_wait_for_rlc_serdes(adev);
5718         }
5719
5720         amdgpu_gfx_rlc_exit_safe_mode(adev);
5721 }
5722
5723 static void gfx_v8_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev,
5724                                                       bool enable)
5725 {
5726         uint32_t temp, temp1, data, data1;
5727
5728         temp = data = RREG32(mmRLC_CGCG_CGLS_CTRL);
5729
5730         amdgpu_gfx_rlc_enter_safe_mode(adev);
5731
5732         if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)) {
5733                 temp1 = data1 = RREG32(mmRLC_CGTT_MGCG_OVERRIDE);
5734                 data1 &= ~RLC_CGTT_MGCG_OVERRIDE__CGCG_MASK;
5735                 if (temp1 != data1)
5736                         WREG32(mmRLC_CGTT_MGCG_OVERRIDE, data1);
5737
5738                 /* : wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */
5739                 gfx_v8_0_wait_for_rlc_serdes(adev);
5740
5741                 /* 2 - clear cgcg override */
5742                 gfx_v8_0_send_serdes_cmd(adev, BPM_REG_CGCG_OVERRIDE, CLE_BPM_SERDES_CMD);
5743
5744                 /* wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */
5745                 gfx_v8_0_wait_for_rlc_serdes(adev);
5746
5747                 /* 3 - write cmd to set CGLS */
5748                 gfx_v8_0_send_serdes_cmd(adev, BPM_REG_CGLS_EN, SET_BPM_SERDES_CMD);
5749
5750                 /* 4 - enable cgcg */
5751                 data |= RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
5752
5753                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) {
5754                         /* enable cgls*/
5755                         data |= RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;
5756
5757                         temp1 = data1 = RREG32(mmRLC_CGTT_MGCG_OVERRIDE);
5758                         data1 &= ~RLC_CGTT_MGCG_OVERRIDE__CGLS_MASK;
5759
5760                         if (temp1 != data1)
5761                                 WREG32(mmRLC_CGTT_MGCG_OVERRIDE, data1);
5762                 } else {
5763                         data &= ~RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;
5764                 }
5765
5766                 if (temp != data)
5767                         WREG32(mmRLC_CGCG_CGLS_CTRL, data);
5768
5769                 /* 5 enable cntx_empty_int_enable/cntx_busy_int_enable/
5770                  * Cmp_busy/GFX_Idle interrupts
5771                  */
5772                 gfx_v8_0_enable_gui_idle_interrupt(adev, true);
5773         } else {
5774                 /* disable cntx_empty_int_enable & GFX Idle interrupt */
5775                 gfx_v8_0_enable_gui_idle_interrupt(adev, false);
5776
5777                 /* TEST CGCG */
5778                 temp1 = data1 = RREG32(mmRLC_CGTT_MGCG_OVERRIDE);
5779                 data1 |= (RLC_CGTT_MGCG_OVERRIDE__CGCG_MASK |
5780                                 RLC_CGTT_MGCG_OVERRIDE__CGLS_MASK);
5781                 if (temp1 != data1)
5782                         WREG32(mmRLC_CGTT_MGCG_OVERRIDE, data1);
5783
5784                 /* read gfx register to wake up cgcg */
5785                 RREG32(mmCB_CGTT_SCLK_CTRL);
5786                 RREG32(mmCB_CGTT_SCLK_CTRL);
5787                 RREG32(mmCB_CGTT_SCLK_CTRL);
5788                 RREG32(mmCB_CGTT_SCLK_CTRL);
5789
5790                 /* wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */
5791                 gfx_v8_0_wait_for_rlc_serdes(adev);
5792
5793                 /* write cmd to Set CGCG Override */
5794                 gfx_v8_0_send_serdes_cmd(adev, BPM_REG_CGCG_OVERRIDE, SET_BPM_SERDES_CMD);
5795
5796                 /* wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */
5797                 gfx_v8_0_wait_for_rlc_serdes(adev);
5798
5799                 /* write cmd to Clear CGLS */
5800                 gfx_v8_0_send_serdes_cmd(adev, BPM_REG_CGLS_EN, CLE_BPM_SERDES_CMD);
5801
5802                 /* disable cgcg, cgls should be disabled too. */
5803                 data &= ~(RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK |
5804                           RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK);
5805                 if (temp != data)
5806                         WREG32(mmRLC_CGCG_CGLS_CTRL, data);
5807                 /* enable interrupts again for PG */
5808                 gfx_v8_0_enable_gui_idle_interrupt(adev, true);
5809         }
5810
5811         gfx_v8_0_wait_for_rlc_serdes(adev);
5812
5813         amdgpu_gfx_rlc_exit_safe_mode(adev);
5814 }
5815 static int gfx_v8_0_update_gfx_clock_gating(struct amdgpu_device *adev,
5816                                             bool enable)
5817 {
5818         if (enable) {
5819                 /* CGCG/CGLS should be enabled after MGCG/MGLS/TS(CG/LS)
5820                  * ===  MGCG + MGLS + TS(CG/LS) ===
5821                  */
5822                 gfx_v8_0_update_medium_grain_clock_gating(adev, enable);
5823                 gfx_v8_0_update_coarse_grain_clock_gating(adev, enable);
5824         } else {
5825                 /* CGCG/CGLS should be disabled before MGCG/MGLS/TS(CG/LS)
5826                  * ===  CGCG + CGLS ===
5827                  */
5828                 gfx_v8_0_update_coarse_grain_clock_gating(adev, enable);
5829                 gfx_v8_0_update_medium_grain_clock_gating(adev, enable);
5830         }
5831         return 0;
5832 }
5833
5834 static int gfx_v8_0_tonga_update_gfx_clock_gating(struct amdgpu_device *adev,
5835                                           enum amd_clockgating_state state)
5836 {
5837         uint32_t msg_id, pp_state = 0;
5838         uint32_t pp_support_state = 0;
5839
5840         if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_CGCG | AMD_CG_SUPPORT_GFX_CGLS)) {
5841                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) {
5842                         pp_support_state = PP_STATE_SUPPORT_LS;
5843                         pp_state = PP_STATE_LS;
5844                 }
5845                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG) {
5846                         pp_support_state |= PP_STATE_SUPPORT_CG;
5847                         pp_state |= PP_STATE_CG;
5848                 }
5849                 if (state == AMD_CG_STATE_UNGATE)
5850                         pp_state = 0;
5851
5852                 msg_id = PP_CG_MSG_ID(PP_GROUP_GFX,
5853                                 PP_BLOCK_GFX_CG,
5854                                 pp_support_state,
5855                                 pp_state);
5856                 amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
5857         }
5858
5859         if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_MGCG | AMD_CG_SUPPORT_GFX_MGLS)) {
5860                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) {
5861                         pp_support_state = PP_STATE_SUPPORT_LS;
5862                         pp_state = PP_STATE_LS;
5863                 }
5864
5865                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG) {
5866                         pp_support_state |= PP_STATE_SUPPORT_CG;
5867                         pp_state |= PP_STATE_CG;
5868                 }
5869
5870                 if (state == AMD_CG_STATE_UNGATE)
5871                         pp_state = 0;
5872
5873                 msg_id = PP_CG_MSG_ID(PP_GROUP_GFX,
5874                                 PP_BLOCK_GFX_MG,
5875                                 pp_support_state,
5876                                 pp_state);
5877                 amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
5878         }
5879
5880         return 0;
5881 }
5882
5883 static int gfx_v8_0_polaris_update_gfx_clock_gating(struct amdgpu_device *adev,
5884                                           enum amd_clockgating_state state)
5885 {
5886
5887         uint32_t msg_id, pp_state = 0;
5888         uint32_t pp_support_state = 0;
5889
5890         if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_CGCG | AMD_CG_SUPPORT_GFX_CGLS)) {
5891                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) {
5892                         pp_support_state = PP_STATE_SUPPORT_LS;
5893                         pp_state = PP_STATE_LS;
5894                 }
5895                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG) {
5896                         pp_support_state |= PP_STATE_SUPPORT_CG;
5897                         pp_state |= PP_STATE_CG;
5898                 }
5899                 if (state == AMD_CG_STATE_UNGATE)
5900                         pp_state = 0;
5901
5902                 msg_id = PP_CG_MSG_ID(PP_GROUP_GFX,
5903                                 PP_BLOCK_GFX_CG,
5904                                 pp_support_state,
5905                                 pp_state);
5906                 amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
5907         }
5908
5909         if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_3D_CGCG | AMD_CG_SUPPORT_GFX_3D_CGLS)) {
5910                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS) {
5911                         pp_support_state = PP_STATE_SUPPORT_LS;
5912                         pp_state = PP_STATE_LS;
5913                 }
5914                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG) {
5915                         pp_support_state |= PP_STATE_SUPPORT_CG;
5916                         pp_state |= PP_STATE_CG;
5917                 }
5918                 if (state == AMD_CG_STATE_UNGATE)
5919                         pp_state = 0;
5920
5921                 msg_id = PP_CG_MSG_ID(PP_GROUP_GFX,
5922                                 PP_BLOCK_GFX_3D,
5923                                 pp_support_state,
5924                                 pp_state);
5925                 amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
5926         }
5927
5928         if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_MGCG | AMD_CG_SUPPORT_GFX_MGLS)) {
5929                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) {
5930                         pp_support_state = PP_STATE_SUPPORT_LS;
5931                         pp_state = PP_STATE_LS;
5932                 }
5933
5934                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG) {
5935                         pp_support_state |= PP_STATE_SUPPORT_CG;
5936                         pp_state |= PP_STATE_CG;
5937                 }
5938
5939                 if (state == AMD_CG_STATE_UNGATE)
5940                         pp_state = 0;
5941
5942                 msg_id = PP_CG_MSG_ID(PP_GROUP_GFX,
5943                                 PP_BLOCK_GFX_MG,
5944                                 pp_support_state,
5945                                 pp_state);
5946                 amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
5947         }
5948
5949         if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS) {
5950                 pp_support_state = PP_STATE_SUPPORT_LS;
5951
5952                 if (state == AMD_CG_STATE_UNGATE)
5953                         pp_state = 0;
5954                 else
5955                         pp_state = PP_STATE_LS;
5956
5957                 msg_id = PP_CG_MSG_ID(PP_GROUP_GFX,
5958                                 PP_BLOCK_GFX_RLC,
5959                                 pp_support_state,
5960                                 pp_state);
5961                 amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
5962         }
5963
5964         if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) {
5965                 pp_support_state = PP_STATE_SUPPORT_LS;
5966
5967                 if (state == AMD_CG_STATE_UNGATE)
5968                         pp_state = 0;
5969                 else
5970                         pp_state = PP_STATE_LS;
5971                 msg_id = PP_CG_MSG_ID(PP_GROUP_GFX,
5972                         PP_BLOCK_GFX_CP,
5973                         pp_support_state,
5974                         pp_state);
5975                 amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
5976         }
5977
5978         return 0;
5979 }
5980
5981 static int gfx_v8_0_set_clockgating_state(void *handle,
5982                                           enum amd_clockgating_state state)
5983 {
5984         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5985
5986         if (amdgpu_sriov_vf(adev))
5987                 return 0;
5988
5989         switch (adev->asic_type) {
5990         case CHIP_FIJI:
5991         case CHIP_CARRIZO:
5992         case CHIP_STONEY:
5993                 gfx_v8_0_update_gfx_clock_gating(adev,
5994                                                  state == AMD_CG_STATE_GATE);
5995                 break;
5996         case CHIP_TONGA:
5997                 gfx_v8_0_tonga_update_gfx_clock_gating(adev, state);
5998                 break;
5999         case CHIP_POLARIS10:
6000         case CHIP_POLARIS11:
6001         case CHIP_POLARIS12:
6002         case CHIP_VEGAM:
6003                 gfx_v8_0_polaris_update_gfx_clock_gating(adev, state);
6004                 break;
6005         default:
6006                 break;
6007         }
6008         return 0;
6009 }
6010
6011 static u64 gfx_v8_0_ring_get_rptr(struct amdgpu_ring *ring)
6012 {
6013         return *ring->rptr_cpu_addr;
6014 }
6015
6016 static u64 gfx_v8_0_ring_get_wptr_gfx(struct amdgpu_ring *ring)
6017 {
6018         struct amdgpu_device *adev = ring->adev;
6019
6020         if (ring->use_doorbell)
6021                 /* XXX check if swapping is necessary on BE */
6022                 return *ring->wptr_cpu_addr;
6023         else
6024                 return RREG32(mmCP_RB0_WPTR);
6025 }
6026
6027 static void gfx_v8_0_ring_set_wptr_gfx(struct amdgpu_ring *ring)
6028 {
6029         struct amdgpu_device *adev = ring->adev;
6030
6031         if (ring->use_doorbell) {
6032                 /* XXX check if swapping is necessary on BE */
6033                 *ring->wptr_cpu_addr = lower_32_bits(ring->wptr);
6034                 WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
6035         } else {
6036                 WREG32(mmCP_RB0_WPTR, lower_32_bits(ring->wptr));
6037                 (void)RREG32(mmCP_RB0_WPTR);
6038         }
6039 }
6040
6041 static void gfx_v8_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
6042 {
6043         u32 ref_and_mask, reg_mem_engine;
6044
6045         if ((ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) ||
6046             (ring->funcs->type == AMDGPU_RING_TYPE_KIQ)) {
6047                 switch (ring->me) {
6048                 case 1:
6049                         ref_and_mask = GPU_HDP_FLUSH_DONE__CP2_MASK << ring->pipe;
6050                         break;
6051                 case 2:
6052                         ref_and_mask = GPU_HDP_FLUSH_DONE__CP6_MASK << ring->pipe;
6053                         break;
6054                 default:
6055                         return;
6056                 }
6057                 reg_mem_engine = 0;
6058         } else {
6059                 ref_and_mask = GPU_HDP_FLUSH_DONE__CP0_MASK;
6060                 reg_mem_engine = WAIT_REG_MEM_ENGINE(1); /* pfp */
6061         }
6062
6063         amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
6064         amdgpu_ring_write(ring, (WAIT_REG_MEM_OPERATION(1) | /* write, wait, write */
6065                                  WAIT_REG_MEM_FUNCTION(3) |  /* == */
6066                                  reg_mem_engine));
6067         amdgpu_ring_write(ring, mmGPU_HDP_FLUSH_REQ);
6068         amdgpu_ring_write(ring, mmGPU_HDP_FLUSH_DONE);
6069         amdgpu_ring_write(ring, ref_and_mask);
6070         amdgpu_ring_write(ring, ref_and_mask);
6071         amdgpu_ring_write(ring, 0x20); /* poll interval */
6072 }
6073
6074 static void gfx_v8_0_ring_emit_vgt_flush(struct amdgpu_ring *ring)
6075 {
6076         amdgpu_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE, 0));
6077         amdgpu_ring_write(ring, EVENT_TYPE(VS_PARTIAL_FLUSH) |
6078                 EVENT_INDEX(4));
6079
6080         amdgpu_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE, 0));
6081         amdgpu_ring_write(ring, EVENT_TYPE(VGT_FLUSH) |
6082                 EVENT_INDEX(0));
6083 }
6084
6085 static void gfx_v8_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
6086                                         struct amdgpu_job *job,
6087                                         struct amdgpu_ib *ib,
6088                                         uint32_t flags)
6089 {
6090         unsigned vmid = AMDGPU_JOB_GET_VMID(job);
6091         u32 header, control = 0;
6092
6093         if (ib->flags & AMDGPU_IB_FLAG_CE)
6094                 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
6095         else
6096                 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
6097
6098         control |= ib->length_dw | (vmid << 24);
6099
6100         if (amdgpu_sriov_vf(ring->adev) && (ib->flags & AMDGPU_IB_FLAG_PREEMPT)) {
6101                 control |= INDIRECT_BUFFER_PRE_ENB(1);
6102
6103                 if (!(ib->flags & AMDGPU_IB_FLAG_CE) && vmid)
6104                         gfx_v8_0_ring_emit_de_meta(ring);
6105         }
6106
6107         amdgpu_ring_write(ring, header);
6108         amdgpu_ring_write(ring,
6109 #ifdef __BIG_ENDIAN
6110                           (2 << 0) |
6111 #endif
6112                           (ib->gpu_addr & 0xFFFFFFFC));
6113         amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
6114         amdgpu_ring_write(ring, control);
6115 }
6116
6117 static void gfx_v8_0_ring_emit_ib_compute(struct amdgpu_ring *ring,
6118                                           struct amdgpu_job *job,
6119                                           struct amdgpu_ib *ib,
6120                                           uint32_t flags)
6121 {
6122         unsigned vmid = AMDGPU_JOB_GET_VMID(job);
6123         u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24);
6124
6125         /* Currently, there is a high possibility to get wave ID mismatch
6126          * between ME and GDS, leading to a hw deadlock, because ME generates
6127          * different wave IDs than the GDS expects. This situation happens
6128          * randomly when at least 5 compute pipes use GDS ordered append.
6129          * The wave IDs generated by ME are also wrong after suspend/resume.
6130          * Those are probably bugs somewhere else in the kernel driver.
6131          *
6132          * Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and
6133          * GDS to 0 for this ring (me/pipe).
6134          */
6135         if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) {
6136                 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
6137                 amdgpu_ring_write(ring, mmGDS_COMPUTE_MAX_WAVE_ID - PACKET3_SET_CONFIG_REG_START);
6138                 amdgpu_ring_write(ring, ring->adev->gds.gds_compute_max_wave_id);
6139         }
6140
6141         amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
6142         amdgpu_ring_write(ring,
6143 #ifdef __BIG_ENDIAN
6144                                 (2 << 0) |
6145 #endif
6146                                 (ib->gpu_addr & 0xFFFFFFFC));
6147         amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
6148         amdgpu_ring_write(ring, control);
6149 }
6150
6151 static void gfx_v8_0_ring_emit_fence_gfx(struct amdgpu_ring *ring, u64 addr,
6152                                          u64 seq, unsigned flags)
6153 {
6154         bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
6155         bool int_sel = flags & AMDGPU_FENCE_FLAG_INT;
6156
6157         /* Workaround for cache flush problems. First send a dummy EOP
6158          * event down the pipe with seq one below.
6159          */
6160         amdgpu_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
6161         amdgpu_ring_write(ring, (EOP_TCL1_ACTION_EN |
6162                                  EOP_TC_ACTION_EN |
6163                                  EOP_TC_WB_ACTION_EN |
6164                                  EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
6165                                  EVENT_INDEX(5)));
6166         amdgpu_ring_write(ring, addr & 0xfffffffc);
6167         amdgpu_ring_write(ring, (upper_32_bits(addr) & 0xffff) |
6168                                 DATA_SEL(1) | INT_SEL(0));
6169         amdgpu_ring_write(ring, lower_32_bits(seq - 1));
6170         amdgpu_ring_write(ring, upper_32_bits(seq - 1));
6171
6172         /* Then send the real EOP event down the pipe:
6173          * EVENT_WRITE_EOP - flush caches, send int */
6174         amdgpu_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
6175         amdgpu_ring_write(ring, (EOP_TCL1_ACTION_EN |
6176                                  EOP_TC_ACTION_EN |
6177                                  EOP_TC_WB_ACTION_EN |
6178                                  EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
6179                                  EVENT_INDEX(5)));
6180         amdgpu_ring_write(ring, addr & 0xfffffffc);
6181         amdgpu_ring_write(ring, (upper_32_bits(addr) & 0xffff) |
6182                           DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0));
6183         amdgpu_ring_write(ring, lower_32_bits(seq));
6184         amdgpu_ring_write(ring, upper_32_bits(seq));
6185
6186 }
6187
6188 static void gfx_v8_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
6189 {
6190         int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
6191         uint32_t seq = ring->fence_drv.sync_seq;
6192         uint64_t addr = ring->fence_drv.gpu_addr;
6193
6194         amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
6195         amdgpu_ring_write(ring, (WAIT_REG_MEM_MEM_SPACE(1) | /* memory */
6196                                  WAIT_REG_MEM_FUNCTION(3) | /* equal */
6197                                  WAIT_REG_MEM_ENGINE(usepfp))); /* pfp or me */
6198         amdgpu_ring_write(ring, addr & 0xfffffffc);
6199         amdgpu_ring_write(ring, upper_32_bits(addr) & 0xffffffff);
6200         amdgpu_ring_write(ring, seq);
6201         amdgpu_ring_write(ring, 0xffffffff);
6202         amdgpu_ring_write(ring, 4); /* poll interval */
6203 }
6204
6205 static void gfx_v8_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
6206                                         unsigned vmid, uint64_t pd_addr)
6207 {
6208         int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
6209
6210         amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
6211
6212         /* wait for the invalidate to complete */
6213         amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
6214         amdgpu_ring_write(ring, (WAIT_REG_MEM_OPERATION(0) | /* wait */
6215                                  WAIT_REG_MEM_FUNCTION(0) |  /* always */
6216                                  WAIT_REG_MEM_ENGINE(0))); /* me */
6217         amdgpu_ring_write(ring, mmVM_INVALIDATE_REQUEST);
6218         amdgpu_ring_write(ring, 0);
6219         amdgpu_ring_write(ring, 0); /* ref */
6220         amdgpu_ring_write(ring, 0); /* mask */
6221         amdgpu_ring_write(ring, 0x20); /* poll interval */
6222
6223         /* compute doesn't have PFP */
6224         if (usepfp) {
6225                 /* sync PFP to ME, otherwise we might get invalid PFP reads */
6226                 amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
6227                 amdgpu_ring_write(ring, 0x0);
6228         }
6229 }
6230
6231 static u64 gfx_v8_0_ring_get_wptr_compute(struct amdgpu_ring *ring)
6232 {
6233         return *ring->wptr_cpu_addr;
6234 }
6235
6236 static void gfx_v8_0_ring_set_wptr_compute(struct amdgpu_ring *ring)
6237 {
6238         struct amdgpu_device *adev = ring->adev;
6239
6240         /* XXX check if swapping is necessary on BE */
6241         *ring->wptr_cpu_addr = lower_32_bits(ring->wptr);
6242         WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
6243 }
6244
6245 static void gfx_v8_0_ring_emit_fence_compute(struct amdgpu_ring *ring,
6246                                              u64 addr, u64 seq,
6247                                              unsigned flags)
6248 {
6249         bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
6250         bool int_sel = flags & AMDGPU_FENCE_FLAG_INT;
6251
6252         /* RELEASE_MEM - flush caches, send int */
6253         amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5));
6254         amdgpu_ring_write(ring, (EOP_TCL1_ACTION_EN |
6255                                  EOP_TC_ACTION_EN |
6256                                  EOP_TC_WB_ACTION_EN |
6257                                  EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
6258                                  EVENT_INDEX(5)));
6259         amdgpu_ring_write(ring, DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0));
6260         amdgpu_ring_write(ring, addr & 0xfffffffc);
6261         amdgpu_ring_write(ring, upper_32_bits(addr));
6262         amdgpu_ring_write(ring, lower_32_bits(seq));
6263         amdgpu_ring_write(ring, upper_32_bits(seq));
6264 }
6265
6266 static void gfx_v8_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr,
6267                                          u64 seq, unsigned int flags)
6268 {
6269         /* we only allocate 32bit for each seq wb address */
6270         BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
6271
6272         /* write fence seq to the "addr" */
6273         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
6274         amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
6275                                  WRITE_DATA_DST_SEL(5) | WR_CONFIRM));
6276         amdgpu_ring_write(ring, lower_32_bits(addr));
6277         amdgpu_ring_write(ring, upper_32_bits(addr));
6278         amdgpu_ring_write(ring, lower_32_bits(seq));
6279
6280         if (flags & AMDGPU_FENCE_FLAG_INT) {
6281                 /* set register to trigger INT */
6282                 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
6283                 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
6284                                          WRITE_DATA_DST_SEL(0) | WR_CONFIRM));
6285                 amdgpu_ring_write(ring, mmCPC_INT_STATUS);
6286                 amdgpu_ring_write(ring, 0);
6287                 amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */
6288         }
6289 }
6290
6291 static void gfx_v8_ring_emit_sb(struct amdgpu_ring *ring)
6292 {
6293         amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
6294         amdgpu_ring_write(ring, 0);
6295 }
6296
6297 static void gfx_v8_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags)
6298 {
6299         uint32_t dw2 = 0;
6300
6301         if (amdgpu_sriov_vf(ring->adev))
6302                 gfx_v8_0_ring_emit_ce_meta(ring);
6303
6304         dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */
6305         if (flags & AMDGPU_HAVE_CTX_SWITCH) {
6306                 gfx_v8_0_ring_emit_vgt_flush(ring);
6307                 /* set load_global_config & load_global_uconfig */
6308                 dw2 |= 0x8001;
6309                 /* set load_cs_sh_regs */
6310                 dw2 |= 0x01000000;
6311                 /* set load_per_context_state & load_gfx_sh_regs for GFX */
6312                 dw2 |= 0x10002;
6313
6314                 /* set load_ce_ram if preamble presented */
6315                 if (AMDGPU_PREAMBLE_IB_PRESENT & flags)
6316                         dw2 |= 0x10000000;
6317         } else {
6318                 /* still load_ce_ram if this is the first time preamble presented
6319                  * although there is no context switch happens.
6320                  */
6321                 if (AMDGPU_PREAMBLE_IB_PRESENT_FIRST & flags)
6322                         dw2 |= 0x10000000;
6323         }
6324
6325         amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
6326         amdgpu_ring_write(ring, dw2);
6327         amdgpu_ring_write(ring, 0);
6328 }
6329
6330 static unsigned gfx_v8_0_ring_emit_init_cond_exec(struct amdgpu_ring *ring)
6331 {
6332         unsigned ret;
6333
6334         amdgpu_ring_write(ring, PACKET3(PACKET3_COND_EXEC, 3));
6335         amdgpu_ring_write(ring, lower_32_bits(ring->cond_exe_gpu_addr));
6336         amdgpu_ring_write(ring, upper_32_bits(ring->cond_exe_gpu_addr));
6337         amdgpu_ring_write(ring, 0); /* discard following DWs if *cond_exec_gpu_addr==0 */
6338         ret = ring->wptr & ring->buf_mask;
6339         amdgpu_ring_write(ring, 0x55aa55aa); /* patch dummy value later */
6340         return ret;
6341 }
6342
6343 static void gfx_v8_0_ring_emit_patch_cond_exec(struct amdgpu_ring *ring, unsigned offset)
6344 {
6345         unsigned cur;
6346
6347         BUG_ON(offset > ring->buf_mask);
6348         BUG_ON(ring->ring[offset] != 0x55aa55aa);
6349
6350         cur = (ring->wptr & ring->buf_mask) - 1;
6351         if (likely(cur > offset))
6352                 ring->ring[offset] = cur - offset;
6353         else
6354                 ring->ring[offset] = (ring->ring_size >> 2) - offset + cur;
6355 }
6356
6357 static void gfx_v8_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg,
6358                                     uint32_t reg_val_offs)
6359 {
6360         struct amdgpu_device *adev = ring->adev;
6361
6362         amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));
6363         amdgpu_ring_write(ring, 0 |     /* src: register*/
6364                                 (5 << 8) |      /* dst: memory */
6365                                 (1 << 20));     /* write confirm */
6366         amdgpu_ring_write(ring, reg);
6367         amdgpu_ring_write(ring, 0);
6368         amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +
6369                                 reg_val_offs * 4));
6370         amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +
6371                                 reg_val_offs * 4));
6372 }
6373
6374 static void gfx_v8_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg,
6375                                   uint32_t val)
6376 {
6377         uint32_t cmd;
6378
6379         switch (ring->funcs->type) {
6380         case AMDGPU_RING_TYPE_GFX:
6381                 cmd = WRITE_DATA_ENGINE_SEL(1) | WR_CONFIRM;
6382                 break;
6383         case AMDGPU_RING_TYPE_KIQ:
6384                 cmd = 1 << 16; /* no inc addr */
6385                 break;
6386         default:
6387                 cmd = WR_CONFIRM;
6388                 break;
6389         }
6390
6391         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
6392         amdgpu_ring_write(ring, cmd);
6393         amdgpu_ring_write(ring, reg);
6394         amdgpu_ring_write(ring, 0);
6395         amdgpu_ring_write(ring, val);
6396 }
6397
6398 static void gfx_v8_0_ring_soft_recovery(struct amdgpu_ring *ring, unsigned vmid)
6399 {
6400         struct amdgpu_device *adev = ring->adev;
6401         uint32_t value = 0;
6402
6403         value = REG_SET_FIELD(value, SQ_CMD, CMD, 0x03);
6404         value = REG_SET_FIELD(value, SQ_CMD, MODE, 0x01);
6405         value = REG_SET_FIELD(value, SQ_CMD, CHECK_VMID, 1);
6406         value = REG_SET_FIELD(value, SQ_CMD, VM_ID, vmid);
6407         WREG32(mmSQ_CMD, value);
6408 }
6409
6410 static void gfx_v8_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev,
6411                                                  enum amdgpu_interrupt_state state)
6412 {
6413         WREG32_FIELD(CP_INT_CNTL_RING0, TIME_STAMP_INT_ENABLE,
6414                      state == AMDGPU_IRQ_STATE_DISABLE ? 0 : 1);
6415 }
6416
6417 static void gfx_v8_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev,
6418                                                      int me, int pipe,
6419                                                      enum amdgpu_interrupt_state state)
6420 {
6421         u32 mec_int_cntl, mec_int_cntl_reg;
6422
6423         /*
6424          * amdgpu controls only the first MEC. That's why this function only
6425          * handles the setting of interrupts for this specific MEC. All other
6426          * pipes' interrupts are set by amdkfd.
6427          */
6428
6429         if (me == 1) {
6430                 switch (pipe) {
6431                 case 0:
6432                         mec_int_cntl_reg = mmCP_ME1_PIPE0_INT_CNTL;
6433                         break;
6434                 case 1:
6435                         mec_int_cntl_reg = mmCP_ME1_PIPE1_INT_CNTL;
6436                         break;
6437                 case 2:
6438                         mec_int_cntl_reg = mmCP_ME1_PIPE2_INT_CNTL;
6439                         break;
6440                 case 3:
6441                         mec_int_cntl_reg = mmCP_ME1_PIPE3_INT_CNTL;
6442                         break;
6443                 default:
6444                         DRM_DEBUG("invalid pipe %d\n", pipe);
6445                         return;
6446                 }
6447         } else {
6448                 DRM_DEBUG("invalid me %d\n", me);
6449                 return;
6450         }
6451
6452         switch (state) {
6453         case AMDGPU_IRQ_STATE_DISABLE:
6454                 mec_int_cntl = RREG32(mec_int_cntl_reg);
6455                 mec_int_cntl &= ~CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE_MASK;
6456                 WREG32(mec_int_cntl_reg, mec_int_cntl);
6457                 break;
6458         case AMDGPU_IRQ_STATE_ENABLE:
6459                 mec_int_cntl = RREG32(mec_int_cntl_reg);
6460                 mec_int_cntl |= CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE_MASK;
6461                 WREG32(mec_int_cntl_reg, mec_int_cntl);
6462                 break;
6463         default:
6464                 break;
6465         }
6466 }
6467
6468 static int gfx_v8_0_set_priv_reg_fault_state(struct amdgpu_device *adev,
6469                                              struct amdgpu_irq_src *source,
6470                                              unsigned type,
6471                                              enum amdgpu_interrupt_state state)
6472 {
6473         WREG32_FIELD(CP_INT_CNTL_RING0, PRIV_REG_INT_ENABLE,
6474                      state == AMDGPU_IRQ_STATE_DISABLE ? 0 : 1);
6475
6476         return 0;
6477 }
6478
6479 static int gfx_v8_0_set_priv_inst_fault_state(struct amdgpu_device *adev,
6480                                               struct amdgpu_irq_src *source,
6481                                               unsigned type,
6482                                               enum amdgpu_interrupt_state state)
6483 {
6484         WREG32_FIELD(CP_INT_CNTL_RING0, PRIV_INSTR_INT_ENABLE,
6485                      state == AMDGPU_IRQ_STATE_DISABLE ? 0 : 1);
6486
6487         return 0;
6488 }
6489
6490 static int gfx_v8_0_set_eop_interrupt_state(struct amdgpu_device *adev,
6491                                             struct amdgpu_irq_src *src,
6492                                             unsigned type,
6493                                             enum amdgpu_interrupt_state state)
6494 {
6495         switch (type) {
6496         case AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP:
6497                 gfx_v8_0_set_gfx_eop_interrupt_state(adev, state);
6498                 break;
6499         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP:
6500                 gfx_v8_0_set_compute_eop_interrupt_state(adev, 1, 0, state);
6501                 break;
6502         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP:
6503                 gfx_v8_0_set_compute_eop_interrupt_state(adev, 1, 1, state);
6504                 break;
6505         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP:
6506                 gfx_v8_0_set_compute_eop_interrupt_state(adev, 1, 2, state);
6507                 break;
6508         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP:
6509                 gfx_v8_0_set_compute_eop_interrupt_state(adev, 1, 3, state);
6510                 break;
6511         case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE0_EOP:
6512                 gfx_v8_0_set_compute_eop_interrupt_state(adev, 2, 0, state);
6513                 break;
6514         case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE1_EOP:
6515                 gfx_v8_0_set_compute_eop_interrupt_state(adev, 2, 1, state);
6516                 break;
6517         case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE2_EOP:
6518                 gfx_v8_0_set_compute_eop_interrupt_state(adev, 2, 2, state);
6519                 break;
6520         case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE3_EOP:
6521                 gfx_v8_0_set_compute_eop_interrupt_state(adev, 2, 3, state);
6522                 break;
6523         default:
6524                 break;
6525         }
6526         return 0;
6527 }
6528
6529 static int gfx_v8_0_set_cp_ecc_int_state(struct amdgpu_device *adev,
6530                                          struct amdgpu_irq_src *source,
6531                                          unsigned int type,
6532                                          enum amdgpu_interrupt_state state)
6533 {
6534         int enable_flag;
6535
6536         switch (state) {
6537         case AMDGPU_IRQ_STATE_DISABLE:
6538                 enable_flag = 0;
6539                 break;
6540
6541         case AMDGPU_IRQ_STATE_ENABLE:
6542                 enable_flag = 1;
6543                 break;
6544
6545         default:
6546                 return -EINVAL;
6547         }
6548
6549         WREG32_FIELD(CP_INT_CNTL, CP_ECC_ERROR_INT_ENABLE, enable_flag);
6550         WREG32_FIELD(CP_INT_CNTL_RING0, CP_ECC_ERROR_INT_ENABLE, enable_flag);
6551         WREG32_FIELD(CP_INT_CNTL_RING1, CP_ECC_ERROR_INT_ENABLE, enable_flag);
6552         WREG32_FIELD(CP_INT_CNTL_RING2, CP_ECC_ERROR_INT_ENABLE, enable_flag);
6553         WREG32_FIELD(CPC_INT_CNTL, CP_ECC_ERROR_INT_ENABLE, enable_flag);
6554         WREG32_FIELD(CP_ME1_PIPE0_INT_CNTL, CP_ECC_ERROR_INT_ENABLE,
6555                      enable_flag);
6556         WREG32_FIELD(CP_ME1_PIPE1_INT_CNTL, CP_ECC_ERROR_INT_ENABLE,
6557                      enable_flag);
6558         WREG32_FIELD(CP_ME1_PIPE2_INT_CNTL, CP_ECC_ERROR_INT_ENABLE,
6559                      enable_flag);
6560         WREG32_FIELD(CP_ME1_PIPE3_INT_CNTL, CP_ECC_ERROR_INT_ENABLE,
6561                      enable_flag);
6562         WREG32_FIELD(CP_ME2_PIPE0_INT_CNTL, CP_ECC_ERROR_INT_ENABLE,
6563                      enable_flag);
6564         WREG32_FIELD(CP_ME2_PIPE1_INT_CNTL, CP_ECC_ERROR_INT_ENABLE,
6565                      enable_flag);
6566         WREG32_FIELD(CP_ME2_PIPE2_INT_CNTL, CP_ECC_ERROR_INT_ENABLE,
6567                      enable_flag);
6568         WREG32_FIELD(CP_ME2_PIPE3_INT_CNTL, CP_ECC_ERROR_INT_ENABLE,
6569                      enable_flag);
6570
6571         return 0;
6572 }
6573
6574 static int gfx_v8_0_set_sq_int_state(struct amdgpu_device *adev,
6575                                      struct amdgpu_irq_src *source,
6576                                      unsigned int type,
6577                                      enum amdgpu_interrupt_state state)
6578 {
6579         int enable_flag;
6580
6581         switch (state) {
6582         case AMDGPU_IRQ_STATE_DISABLE:
6583                 enable_flag = 1;
6584                 break;
6585
6586         case AMDGPU_IRQ_STATE_ENABLE:
6587                 enable_flag = 0;
6588                 break;
6589
6590         default:
6591                 return -EINVAL;
6592         }
6593
6594         WREG32_FIELD(SQ_INTERRUPT_MSG_CTRL, STALL,
6595                      enable_flag);
6596
6597         return 0;
6598 }
6599
6600 static int gfx_v8_0_eop_irq(struct amdgpu_device *adev,
6601                             struct amdgpu_irq_src *source,
6602                             struct amdgpu_iv_entry *entry)
6603 {
6604         int i;
6605         u8 me_id, pipe_id, queue_id;
6606         struct amdgpu_ring *ring;
6607
6608         DRM_DEBUG("IH: CP EOP\n");
6609         me_id = (entry->ring_id & 0x0c) >> 2;
6610         pipe_id = (entry->ring_id & 0x03) >> 0;
6611         queue_id = (entry->ring_id & 0x70) >> 4;
6612
6613         switch (me_id) {
6614         case 0:
6615                 amdgpu_fence_process(&adev->gfx.gfx_ring[0]);
6616                 break;
6617         case 1:
6618         case 2:
6619                 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
6620                         ring = &adev->gfx.compute_ring[i];
6621                         /* Per-queue interrupt is supported for MEC starting from VI.
6622                           * The interrupt can only be enabled/disabled per pipe instead of per queue.
6623                           */
6624                         if ((ring->me == me_id) && (ring->pipe == pipe_id) && (ring->queue == queue_id))
6625                                 amdgpu_fence_process(ring);
6626                 }
6627                 break;
6628         }
6629         return 0;
6630 }
6631
6632 static void gfx_v8_0_fault(struct amdgpu_device *adev,
6633                            struct amdgpu_iv_entry *entry)
6634 {
6635         u8 me_id, pipe_id, queue_id;
6636         struct amdgpu_ring *ring;
6637         int i;
6638
6639         me_id = (entry->ring_id & 0x0c) >> 2;
6640         pipe_id = (entry->ring_id & 0x03) >> 0;
6641         queue_id = (entry->ring_id & 0x70) >> 4;
6642
6643         switch (me_id) {
6644         case 0:
6645                 drm_sched_fault(&adev->gfx.gfx_ring[0].sched);
6646                 break;
6647         case 1:
6648         case 2:
6649                 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
6650                         ring = &adev->gfx.compute_ring[i];
6651                         if (ring->me == me_id && ring->pipe == pipe_id &&
6652                             ring->queue == queue_id)
6653                                 drm_sched_fault(&ring->sched);
6654                 }
6655                 break;
6656         }
6657 }
6658
6659 static int gfx_v8_0_priv_reg_irq(struct amdgpu_device *adev,
6660                                  struct amdgpu_irq_src *source,
6661                                  struct amdgpu_iv_entry *entry)
6662 {
6663         DRM_ERROR("Illegal register access in command stream\n");
6664         gfx_v8_0_fault(adev, entry);
6665         return 0;
6666 }
6667
6668 static int gfx_v8_0_priv_inst_irq(struct amdgpu_device *adev,
6669                                   struct amdgpu_irq_src *source,
6670                                   struct amdgpu_iv_entry *entry)
6671 {
6672         DRM_ERROR("Illegal instruction in command stream\n");
6673         gfx_v8_0_fault(adev, entry);
6674         return 0;
6675 }
6676
6677 static int gfx_v8_0_cp_ecc_error_irq(struct amdgpu_device *adev,
6678                                      struct amdgpu_irq_src *source,
6679                                      struct amdgpu_iv_entry *entry)
6680 {
6681         DRM_ERROR("CP EDC/ECC error detected.");
6682         return 0;
6683 }
6684
6685 static void gfx_v8_0_parse_sq_irq(struct amdgpu_device *adev, unsigned ih_data,
6686                                   bool from_wq)
6687 {
6688         u32 enc, se_id, sh_id, cu_id;
6689         char type[20];
6690         int sq_edc_source = -1;
6691
6692         enc = REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_CMN, ENCODING);
6693         se_id = REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_CMN, SE_ID);
6694
6695         switch (enc) {
6696                 case 0:
6697                         DRM_INFO("SQ general purpose intr detected:"
6698                                         "se_id %d, immed_overflow %d, host_reg_overflow %d,"
6699                                         "host_cmd_overflow %d, cmd_timestamp %d,"
6700                                         "reg_timestamp %d, thread_trace_buff_full %d,"
6701                                         "wlt %d, thread_trace %d.\n",
6702                                         se_id,
6703                                         REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, IMMED_OVERFLOW),
6704                                         REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, HOST_REG_OVERFLOW),
6705                                         REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, HOST_CMD_OVERFLOW),
6706                                         REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, CMD_TIMESTAMP),
6707                                         REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, REG_TIMESTAMP),
6708                                         REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, THREAD_TRACE_BUF_FULL),
6709                                         REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, WLT),
6710                                         REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, THREAD_TRACE)
6711                                         );
6712                         break;
6713                 case 1:
6714                 case 2:
6715
6716                         cu_id = REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_WAVE, CU_ID);
6717                         sh_id = REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_WAVE, SH_ID);
6718
6719                         /*
6720                          * This function can be called either directly from ISR
6721                          * or from BH in which case we can access SQ_EDC_INFO
6722                          * instance
6723                          */
6724                         if (from_wq) {
6725                                 mutex_lock(&adev->grbm_idx_mutex);
6726                                 gfx_v8_0_select_se_sh(adev, se_id, sh_id, cu_id);
6727
6728                                 sq_edc_source = REG_GET_FIELD(RREG32(mmSQ_EDC_INFO), SQ_EDC_INFO, SOURCE);
6729
6730                                 gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
6731                                 mutex_unlock(&adev->grbm_idx_mutex);
6732                         }
6733
6734                         if (enc == 1)
6735                                 sprintf(type, "instruction intr");
6736                         else
6737                                 sprintf(type, "EDC/ECC error");
6738
6739                         DRM_INFO(
6740                                 "SQ %s detected: "
6741                                         "se_id %d, sh_id %d, cu_id %d, simd_id %d, wave_id %d, vm_id %d "
6742                                         "trap %s, sq_ed_info.source %s.\n",
6743                                         type, se_id, sh_id, cu_id,
6744                                         REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_WAVE, SIMD_ID),
6745                                         REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_WAVE, WAVE_ID),
6746                                         REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_WAVE, VM_ID),
6747                                         REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_WAVE, PRIV) ? "true" : "false",
6748                                         (sq_edc_source != -1) ? sq_edc_source_names[sq_edc_source] : "unavailable"
6749                                 );
6750                         break;
6751                 default:
6752                         DRM_ERROR("SQ invalid encoding type\n.");
6753         }
6754 }
6755
6756 static void gfx_v8_0_sq_irq_work_func(struct work_struct *work)
6757 {
6758
6759         struct amdgpu_device *adev = container_of(work, struct amdgpu_device, gfx.sq_work.work);
6760         struct sq_work *sq_work = container_of(work, struct sq_work, work);
6761
6762         gfx_v8_0_parse_sq_irq(adev, sq_work->ih_data, true);
6763 }
6764
6765 static int gfx_v8_0_sq_irq(struct amdgpu_device *adev,
6766                            struct amdgpu_irq_src *source,
6767                            struct amdgpu_iv_entry *entry)
6768 {
6769         unsigned ih_data = entry->src_data[0];
6770
6771         /*
6772          * Try to submit work so SQ_EDC_INFO can be accessed from
6773          * BH. If previous work submission hasn't finished yet
6774          * just print whatever info is possible directly from the ISR.
6775          */
6776         if (work_pending(&adev->gfx.sq_work.work)) {
6777                 gfx_v8_0_parse_sq_irq(adev, ih_data, false);
6778         } else {
6779                 adev->gfx.sq_work.ih_data = ih_data;
6780                 schedule_work(&adev->gfx.sq_work.work);
6781         }
6782
6783         return 0;
6784 }
6785
6786 static void gfx_v8_0_emit_mem_sync(struct amdgpu_ring *ring)
6787 {
6788         amdgpu_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
6789         amdgpu_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
6790                           PACKET3_TC_ACTION_ENA |
6791                           PACKET3_SH_KCACHE_ACTION_ENA |
6792                           PACKET3_SH_ICACHE_ACTION_ENA |
6793                           PACKET3_TC_WB_ACTION_ENA);  /* CP_COHER_CNTL */
6794         amdgpu_ring_write(ring, 0xffffffff);  /* CP_COHER_SIZE */
6795         amdgpu_ring_write(ring, 0);  /* CP_COHER_BASE */
6796         amdgpu_ring_write(ring, 0x0000000A); /* poll interval */
6797 }
6798
6799 static void gfx_v8_0_emit_mem_sync_compute(struct amdgpu_ring *ring)
6800 {
6801         amdgpu_ring_write(ring, PACKET3(PACKET3_ACQUIRE_MEM, 5));
6802         amdgpu_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
6803                           PACKET3_TC_ACTION_ENA |
6804                           PACKET3_SH_KCACHE_ACTION_ENA |
6805                           PACKET3_SH_ICACHE_ACTION_ENA |
6806                           PACKET3_TC_WB_ACTION_ENA);  /* CP_COHER_CNTL */
6807         amdgpu_ring_write(ring, 0xffffffff);    /* CP_COHER_SIZE */
6808         amdgpu_ring_write(ring, 0xff);          /* CP_COHER_SIZE_HI */
6809         amdgpu_ring_write(ring, 0);             /* CP_COHER_BASE */
6810         amdgpu_ring_write(ring, 0);             /* CP_COHER_BASE_HI */
6811         amdgpu_ring_write(ring, 0x0000000A);    /* poll interval */
6812 }
6813
6814
6815 /* mmSPI_WCL_PIPE_PERCENT_CS[0-7]_DEFAULT values are same */
6816 #define mmSPI_WCL_PIPE_PERCENT_CS_DEFAULT       0x0000007f
6817 static void gfx_v8_0_emit_wave_limit_cs(struct amdgpu_ring *ring,
6818                                         uint32_t pipe, bool enable)
6819 {
6820         uint32_t val;
6821         uint32_t wcl_cs_reg;
6822
6823         val = enable ? 0x1 : mmSPI_WCL_PIPE_PERCENT_CS_DEFAULT;
6824
6825         switch (pipe) {
6826         case 0:
6827                 wcl_cs_reg = mmSPI_WCL_PIPE_PERCENT_CS0;
6828                 break;
6829         case 1:
6830                 wcl_cs_reg = mmSPI_WCL_PIPE_PERCENT_CS1;
6831                 break;
6832         case 2:
6833                 wcl_cs_reg = mmSPI_WCL_PIPE_PERCENT_CS2;
6834                 break;
6835         case 3:
6836                 wcl_cs_reg = mmSPI_WCL_PIPE_PERCENT_CS3;
6837                 break;
6838         default:
6839                 DRM_DEBUG("invalid pipe %d\n", pipe);
6840                 return;
6841         }
6842
6843         amdgpu_ring_emit_wreg(ring, wcl_cs_reg, val);
6844
6845 }
6846
6847 #define mmSPI_WCL_PIPE_PERCENT_GFX_DEFAULT      0x07ffffff
6848 static void gfx_v8_0_emit_wave_limit(struct amdgpu_ring *ring, bool enable)
6849 {
6850         struct amdgpu_device *adev = ring->adev;
6851         uint32_t val;
6852         int i;
6853
6854         /* mmSPI_WCL_PIPE_PERCENT_GFX is 7 bit multiplier register to limit
6855          * number of gfx waves. Setting 5 bit will make sure gfx only gets
6856          * around 25% of gpu resources.
6857          */
6858         val = enable ? 0x1f : mmSPI_WCL_PIPE_PERCENT_GFX_DEFAULT;
6859         amdgpu_ring_emit_wreg(ring, mmSPI_WCL_PIPE_PERCENT_GFX, val);
6860
6861         /* Restrict waves for normal/low priority compute queues as well
6862          * to get best QoS for high priority compute jobs.
6863          *
6864          * amdgpu controls only 1st ME(0-3 CS pipes).
6865          */
6866         for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) {
6867                 if (i != ring->pipe)
6868                         gfx_v8_0_emit_wave_limit_cs(ring, i, enable);
6869
6870         }
6871
6872 }
6873
6874 static const struct amd_ip_funcs gfx_v8_0_ip_funcs = {
6875         .name = "gfx_v8_0",
6876         .early_init = gfx_v8_0_early_init,
6877         .late_init = gfx_v8_0_late_init,
6878         .sw_init = gfx_v8_0_sw_init,
6879         .sw_fini = gfx_v8_0_sw_fini,
6880         .hw_init = gfx_v8_0_hw_init,
6881         .hw_fini = gfx_v8_0_hw_fini,
6882         .suspend = gfx_v8_0_suspend,
6883         .resume = gfx_v8_0_resume,
6884         .is_idle = gfx_v8_0_is_idle,
6885         .wait_for_idle = gfx_v8_0_wait_for_idle,
6886         .check_soft_reset = gfx_v8_0_check_soft_reset,
6887         .pre_soft_reset = gfx_v8_0_pre_soft_reset,
6888         .soft_reset = gfx_v8_0_soft_reset,
6889         .post_soft_reset = gfx_v8_0_post_soft_reset,
6890         .set_clockgating_state = gfx_v8_0_set_clockgating_state,
6891         .set_powergating_state = gfx_v8_0_set_powergating_state,
6892         .get_clockgating_state = gfx_v8_0_get_clockgating_state,
6893 };
6894
6895 static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_gfx = {
6896         .type = AMDGPU_RING_TYPE_GFX,
6897         .align_mask = 0xff,
6898         .nop = PACKET3(PACKET3_NOP, 0x3FFF),
6899         .support_64bit_ptrs = false,
6900         .get_rptr = gfx_v8_0_ring_get_rptr,
6901         .get_wptr = gfx_v8_0_ring_get_wptr_gfx,
6902         .set_wptr = gfx_v8_0_ring_set_wptr_gfx,
6903         .emit_frame_size = /* maximum 215dw if count 16 IBs in */
6904                 5 +  /* COND_EXEC */
6905                 7 +  /* PIPELINE_SYNC */
6906                 VI_FLUSH_GPU_TLB_NUM_WREG * 5 + 9 + /* VM_FLUSH */
6907                 12 +  /* FENCE for VM_FLUSH */
6908                 20 + /* GDS switch */
6909                 4 + /* double SWITCH_BUFFER,
6910                        the first COND_EXEC jump to the place just
6911                            prior to this double SWITCH_BUFFER  */
6912                 5 + /* COND_EXEC */
6913                 7 +      /*     HDP_flush */
6914                 4 +      /*     VGT_flush */
6915                 14 + /* CE_META */
6916                 31 + /* DE_META */
6917                 3 + /* CNTX_CTRL */
6918                 5 + /* HDP_INVL */
6919                 12 + 12 + /* FENCE x2 */
6920                 2 + /* SWITCH_BUFFER */
6921                 5, /* SURFACE_SYNC */
6922         .emit_ib_size = 4, /* gfx_v8_0_ring_emit_ib_gfx */
6923         .emit_ib = gfx_v8_0_ring_emit_ib_gfx,
6924         .emit_fence = gfx_v8_0_ring_emit_fence_gfx,
6925         .emit_pipeline_sync = gfx_v8_0_ring_emit_pipeline_sync,
6926         .emit_vm_flush = gfx_v8_0_ring_emit_vm_flush,
6927         .emit_gds_switch = gfx_v8_0_ring_emit_gds_switch,
6928         .emit_hdp_flush = gfx_v8_0_ring_emit_hdp_flush,
6929         .test_ring = gfx_v8_0_ring_test_ring,
6930         .test_ib = gfx_v8_0_ring_test_ib,
6931         .insert_nop = amdgpu_ring_insert_nop,
6932         .pad_ib = amdgpu_ring_generic_pad_ib,
6933         .emit_switch_buffer = gfx_v8_ring_emit_sb,
6934         .emit_cntxcntl = gfx_v8_ring_emit_cntxcntl,
6935         .init_cond_exec = gfx_v8_0_ring_emit_init_cond_exec,
6936         .patch_cond_exec = gfx_v8_0_ring_emit_patch_cond_exec,
6937         .emit_wreg = gfx_v8_0_ring_emit_wreg,
6938         .soft_recovery = gfx_v8_0_ring_soft_recovery,
6939         .emit_mem_sync = gfx_v8_0_emit_mem_sync,
6940 };
6941
6942 static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_compute = {
6943         .type = AMDGPU_RING_TYPE_COMPUTE,
6944         .align_mask = 0xff,
6945         .nop = PACKET3(PACKET3_NOP, 0x3FFF),
6946         .support_64bit_ptrs = false,
6947         .get_rptr = gfx_v8_0_ring_get_rptr,
6948         .get_wptr = gfx_v8_0_ring_get_wptr_compute,
6949         .set_wptr = gfx_v8_0_ring_set_wptr_compute,
6950         .emit_frame_size =
6951                 20 + /* gfx_v8_0_ring_emit_gds_switch */
6952                 7 + /* gfx_v8_0_ring_emit_hdp_flush */
6953                 5 + /* hdp_invalidate */
6954                 7 + /* gfx_v8_0_ring_emit_pipeline_sync */
6955                 VI_FLUSH_GPU_TLB_NUM_WREG * 5 + 7 + /* gfx_v8_0_ring_emit_vm_flush */
6956                 7 + 7 + 7 + /* gfx_v8_0_ring_emit_fence_compute x3 for user fence, vm fence */
6957                 7 + /* gfx_v8_0_emit_mem_sync_compute */
6958                 5 + /* gfx_v8_0_emit_wave_limit for updating mmSPI_WCL_PIPE_PERCENT_GFX register */
6959                 15, /* for updating 3 mmSPI_WCL_PIPE_PERCENT_CS registers */
6960         .emit_ib_size = 7, /* gfx_v8_0_ring_emit_ib_compute */
6961         .emit_ib = gfx_v8_0_ring_emit_ib_compute,
6962         .emit_fence = gfx_v8_0_ring_emit_fence_compute,
6963         .emit_pipeline_sync = gfx_v8_0_ring_emit_pipeline_sync,
6964         .emit_vm_flush = gfx_v8_0_ring_emit_vm_flush,
6965         .emit_gds_switch = gfx_v8_0_ring_emit_gds_switch,
6966         .emit_hdp_flush = gfx_v8_0_ring_emit_hdp_flush,
6967         .test_ring = gfx_v8_0_ring_test_ring,
6968         .test_ib = gfx_v8_0_ring_test_ib,
6969         .insert_nop = amdgpu_ring_insert_nop,
6970         .pad_ib = amdgpu_ring_generic_pad_ib,
6971         .emit_wreg = gfx_v8_0_ring_emit_wreg,
6972         .emit_mem_sync = gfx_v8_0_emit_mem_sync_compute,
6973         .emit_wave_limit = gfx_v8_0_emit_wave_limit,
6974 };
6975
6976 static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_kiq = {
6977         .type = AMDGPU_RING_TYPE_KIQ,
6978         .align_mask = 0xff,
6979         .nop = PACKET3(PACKET3_NOP, 0x3FFF),
6980         .support_64bit_ptrs = false,
6981         .get_rptr = gfx_v8_0_ring_get_rptr,
6982         .get_wptr = gfx_v8_0_ring_get_wptr_compute,
6983         .set_wptr = gfx_v8_0_ring_set_wptr_compute,
6984         .emit_frame_size =
6985                 20 + /* gfx_v8_0_ring_emit_gds_switch */
6986                 7 + /* gfx_v8_0_ring_emit_hdp_flush */
6987                 5 + /* hdp_invalidate */
6988                 7 + /* gfx_v8_0_ring_emit_pipeline_sync */
6989                 17 + /* gfx_v8_0_ring_emit_vm_flush */
6990                 7 + 7 + 7, /* gfx_v8_0_ring_emit_fence_kiq x3 for user fence, vm fence */
6991         .emit_ib_size = 7, /* gfx_v8_0_ring_emit_ib_compute */
6992         .emit_fence = gfx_v8_0_ring_emit_fence_kiq,
6993         .test_ring = gfx_v8_0_ring_test_ring,
6994         .insert_nop = amdgpu_ring_insert_nop,
6995         .pad_ib = amdgpu_ring_generic_pad_ib,
6996         .emit_rreg = gfx_v8_0_ring_emit_rreg,
6997         .emit_wreg = gfx_v8_0_ring_emit_wreg,
6998 };
6999
7000 static void gfx_v8_0_set_ring_funcs(struct amdgpu_device *adev)
7001 {
7002         int i;
7003
7004         adev->gfx.kiq.ring.funcs = &gfx_v8_0_ring_funcs_kiq;
7005
7006         for (i = 0; i < adev->gfx.num_gfx_rings; i++)
7007                 adev->gfx.gfx_ring[i].funcs = &gfx_v8_0_ring_funcs_gfx;
7008
7009         for (i = 0; i < adev->gfx.num_compute_rings; i++)
7010                 adev->gfx.compute_ring[i].funcs = &gfx_v8_0_ring_funcs_compute;
7011 }
7012
7013 static const struct amdgpu_irq_src_funcs gfx_v8_0_eop_irq_funcs = {
7014         .set = gfx_v8_0_set_eop_interrupt_state,
7015         .process = gfx_v8_0_eop_irq,
7016 };
7017
7018 static const struct amdgpu_irq_src_funcs gfx_v8_0_priv_reg_irq_funcs = {
7019         .set = gfx_v8_0_set_priv_reg_fault_state,
7020         .process = gfx_v8_0_priv_reg_irq,
7021 };
7022
7023 static const struct amdgpu_irq_src_funcs gfx_v8_0_priv_inst_irq_funcs = {
7024         .set = gfx_v8_0_set_priv_inst_fault_state,
7025         .process = gfx_v8_0_priv_inst_irq,
7026 };
7027
7028 static const struct amdgpu_irq_src_funcs gfx_v8_0_cp_ecc_error_irq_funcs = {
7029         .set = gfx_v8_0_set_cp_ecc_int_state,
7030         .process = gfx_v8_0_cp_ecc_error_irq,
7031 };
7032
7033 static const struct amdgpu_irq_src_funcs gfx_v8_0_sq_irq_funcs = {
7034         .set = gfx_v8_0_set_sq_int_state,
7035         .process = gfx_v8_0_sq_irq,
7036 };
7037
7038 static void gfx_v8_0_set_irq_funcs(struct amdgpu_device *adev)
7039 {
7040         adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST;
7041         adev->gfx.eop_irq.funcs = &gfx_v8_0_eop_irq_funcs;
7042
7043         adev->gfx.priv_reg_irq.num_types = 1;
7044         adev->gfx.priv_reg_irq.funcs = &gfx_v8_0_priv_reg_irq_funcs;
7045
7046         adev->gfx.priv_inst_irq.num_types = 1;
7047         adev->gfx.priv_inst_irq.funcs = &gfx_v8_0_priv_inst_irq_funcs;
7048
7049         adev->gfx.cp_ecc_error_irq.num_types = 1;
7050         adev->gfx.cp_ecc_error_irq.funcs = &gfx_v8_0_cp_ecc_error_irq_funcs;
7051
7052         adev->gfx.sq_irq.num_types = 1;
7053         adev->gfx.sq_irq.funcs = &gfx_v8_0_sq_irq_funcs;
7054 }
7055
7056 static void gfx_v8_0_set_rlc_funcs(struct amdgpu_device *adev)
7057 {
7058         adev->gfx.rlc.funcs = &iceland_rlc_funcs;
7059 }
7060
7061 static void gfx_v8_0_set_gds_init(struct amdgpu_device *adev)
7062 {
7063         /* init asci gds info */
7064         adev->gds.gds_size = RREG32(mmGDS_VMID0_SIZE);
7065         adev->gds.gws_size = 64;
7066         adev->gds.oa_size = 16;
7067         adev->gds.gds_compute_max_wave_id = RREG32(mmGDS_COMPUTE_MAX_WAVE_ID);
7068 }
7069
7070 static void gfx_v8_0_set_user_cu_inactive_bitmap(struct amdgpu_device *adev,
7071                                                  u32 bitmap)
7072 {
7073         u32 data;
7074
7075         if (!bitmap)
7076                 return;
7077
7078         data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT;
7079         data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK;
7080
7081         WREG32(mmGC_USER_SHADER_ARRAY_CONFIG, data);
7082 }
7083
7084 static u32 gfx_v8_0_get_cu_active_bitmap(struct amdgpu_device *adev)
7085 {
7086         u32 data, mask;
7087
7088         data =  RREG32(mmCC_GC_SHADER_ARRAY_CONFIG) |
7089                 RREG32(mmGC_USER_SHADER_ARRAY_CONFIG);
7090
7091         mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh);
7092
7093         return ~REG_GET_FIELD(data, CC_GC_SHADER_ARRAY_CONFIG, INACTIVE_CUS) & mask;
7094 }
7095
7096 static void gfx_v8_0_get_cu_info(struct amdgpu_device *adev)
7097 {
7098         int i, j, k, counter, active_cu_number = 0;
7099         u32 mask, bitmap, ao_bitmap, ao_cu_mask = 0;
7100         struct amdgpu_cu_info *cu_info = &adev->gfx.cu_info;
7101         unsigned disable_masks[4 * 2];
7102         u32 ao_cu_num;
7103
7104         memset(cu_info, 0, sizeof(*cu_info));
7105
7106         if (adev->flags & AMD_IS_APU)
7107                 ao_cu_num = 2;
7108         else
7109                 ao_cu_num = adev->gfx.config.max_cu_per_sh;
7110
7111         amdgpu_gfx_parse_disable_cu(disable_masks, 4, 2);
7112
7113         mutex_lock(&adev->grbm_idx_mutex);
7114         for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
7115                 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
7116                         mask = 1;
7117                         ao_bitmap = 0;
7118                         counter = 0;
7119                         gfx_v8_0_select_se_sh(adev, i, j, 0xffffffff);
7120                         if (i < 4 && j < 2)
7121                                 gfx_v8_0_set_user_cu_inactive_bitmap(
7122                                         adev, disable_masks[i * 2 + j]);
7123                         bitmap = gfx_v8_0_get_cu_active_bitmap(adev);
7124                         cu_info->bitmap[i][j] = bitmap;
7125
7126                         for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) {
7127                                 if (bitmap & mask) {
7128                                         if (counter < ao_cu_num)
7129                                                 ao_bitmap |= mask;
7130                                         counter ++;
7131                                 }
7132                                 mask <<= 1;
7133                         }
7134                         active_cu_number += counter;
7135                         if (i < 2 && j < 2)
7136                                 ao_cu_mask |= (ao_bitmap << (i * 16 + j * 8));
7137                         cu_info->ao_cu_bitmap[i][j] = ao_bitmap;
7138                 }
7139         }
7140         gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
7141         mutex_unlock(&adev->grbm_idx_mutex);
7142
7143         cu_info->number = active_cu_number;
7144         cu_info->ao_cu_mask = ao_cu_mask;
7145         cu_info->simd_per_cu = NUM_SIMD_PER_CU;
7146         cu_info->max_waves_per_simd = 10;
7147         cu_info->max_scratch_slots_per_cu = 32;
7148         cu_info->wave_front_size = 64;
7149         cu_info->lds_size = 64;
7150 }
7151
7152 const struct amdgpu_ip_block_version gfx_v8_0_ip_block =
7153 {
7154         .type = AMD_IP_BLOCK_TYPE_GFX,
7155         .major = 8,
7156         .minor = 0,
7157         .rev = 0,
7158         .funcs = &gfx_v8_0_ip_funcs,
7159 };
7160
7161 const struct amdgpu_ip_block_version gfx_v8_1_ip_block =
7162 {
7163         .type = AMD_IP_BLOCK_TYPE_GFX,
7164         .major = 8,
7165         .minor = 1,
7166         .rev = 0,
7167         .funcs = &gfx_v8_0_ip_funcs,
7168 };
7169
7170 static void gfx_v8_0_ring_emit_ce_meta(struct amdgpu_ring *ring)
7171 {
7172         uint64_t ce_payload_addr;
7173         int cnt_ce;
7174         union {
7175                 struct vi_ce_ib_state regular;
7176                 struct vi_ce_ib_state_chained_ib chained;
7177         } ce_payload = {};
7178
7179         if (ring->adev->virt.chained_ib_support) {
7180                 ce_payload_addr = amdgpu_csa_vaddr(ring->adev) +
7181                         offsetof(struct vi_gfx_meta_data_chained_ib, ce_payload);
7182                 cnt_ce = (sizeof(ce_payload.chained) >> 2) + 4 - 2;
7183         } else {
7184                 ce_payload_addr = amdgpu_csa_vaddr(ring->adev) +
7185                         offsetof(struct vi_gfx_meta_data, ce_payload);
7186                 cnt_ce = (sizeof(ce_payload.regular) >> 2) + 4 - 2;
7187         }
7188
7189         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt_ce));
7190         amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(2) |
7191                                 WRITE_DATA_DST_SEL(8) |
7192                                 WR_CONFIRM) |
7193                                 WRITE_DATA_CACHE_POLICY(0));
7194         amdgpu_ring_write(ring, lower_32_bits(ce_payload_addr));
7195         amdgpu_ring_write(ring, upper_32_bits(ce_payload_addr));
7196         amdgpu_ring_write_multiple(ring, (void *)&ce_payload, cnt_ce - 2);
7197 }
7198
7199 static void gfx_v8_0_ring_emit_de_meta(struct amdgpu_ring *ring)
7200 {
7201         uint64_t de_payload_addr, gds_addr, csa_addr;
7202         int cnt_de;
7203         union {
7204                 struct vi_de_ib_state regular;
7205                 struct vi_de_ib_state_chained_ib chained;
7206         } de_payload = {};
7207
7208         csa_addr = amdgpu_csa_vaddr(ring->adev);
7209         gds_addr = csa_addr + 4096;
7210         if (ring->adev->virt.chained_ib_support) {
7211                 de_payload.chained.gds_backup_addrlo = lower_32_bits(gds_addr);
7212                 de_payload.chained.gds_backup_addrhi = upper_32_bits(gds_addr);
7213                 de_payload_addr = csa_addr + offsetof(struct vi_gfx_meta_data_chained_ib, de_payload);
7214                 cnt_de = (sizeof(de_payload.chained) >> 2) + 4 - 2;
7215         } else {
7216                 de_payload.regular.gds_backup_addrlo = lower_32_bits(gds_addr);
7217                 de_payload.regular.gds_backup_addrhi = upper_32_bits(gds_addr);
7218                 de_payload_addr = csa_addr + offsetof(struct vi_gfx_meta_data, de_payload);
7219                 cnt_de = (sizeof(de_payload.regular) >> 2) + 4 - 2;
7220         }
7221
7222         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt_de));
7223         amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
7224                                 WRITE_DATA_DST_SEL(8) |
7225                                 WR_CONFIRM) |
7226                                 WRITE_DATA_CACHE_POLICY(0));
7227         amdgpu_ring_write(ring, lower_32_bits(de_payload_addr));
7228         amdgpu_ring_write(ring, upper_32_bits(de_payload_addr));
7229         amdgpu_ring_write_multiple(ring, (void *)&de_payload, cnt_de - 2);
7230 }