drivers/gpu/drm/amd/amdgpu/si.c

   1 /*
   2  * Copyright 2015 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23
  24 #include <linux/firmware.h>
  25 #include <linux/slab.h>
  26 #include <linux/module.h>
  27 #include <linux/pci.h>
  28
  29 #include "amdgpu.h"
  30 #include "amdgpu_atombios.h"
  31 #include "amdgpu_ih.h"
  32 #include "amdgpu_uvd.h"
  33 #include "amdgpu_vce.h"
  34 #include "atom.h"
  35 #include "amd_pcie.h"
  36 #include "si_dpm.h"
  37 #include "sid.h"
  38 #include "si_ih.h"
  39 #include "gfx_v6_0.h"
  40 #include "gmc_v6_0.h"
  41 #include "si_dma.h"
  42 #include "dce_v6_0.h"
  43 #include "si.h"
  44 #include "uvd_v3_1.h"
  45 #include "dce_virtual.h"
  46 #include "gca/gfx_6_0_d.h"
  47 #include "oss/oss_1_0_d.h"
  48 #include "oss/oss_1_0_sh_mask.h"
  49 #include "gmc/gmc_6_0_d.h"
  50 #include "dce/dce_6_0_d.h"
  51 #include "uvd/uvd_4_0_d.h"
  52 #include "bif/bif_3_0_d.h"
  53 #include "bif/bif_3_0_sh_mask.h"
  54
  55 static const u32 tahiti_golden_registers[] =
  56 {
  57         mmAZALIA_SCLK_CONTROL, 0x00000030, 0x00000011,
  58         mmCB_HW_CONTROL, 0x00010000, 0x00018208,
  59         mmDB_DEBUG, 0xffffffff, 0x00000000,
  60         mmDB_DEBUG2, 0xf00fffff, 0x00000400,
  61         mmDB_DEBUG3, 0x0002021c, 0x00020200,
  62         mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
  63         0x340c, 0x000000c0, 0x00800040,
  64         0x360c, 0x000000c0, 0x00800040,
  65         mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
  66         mmFBC_MISC, 0x00200000, 0x50100000,
  67         mmDIG0_HDMI_CONTROL, 0x31000311, 0x00000011,
  68         mmMC_ARB_WTM_CNTL_RD, 0x00000003, 0x000007ff,
  69         mmMC_XPB_P2P_BAR_CFG, 0x000007ff, 0x00000000,
  70         mmPA_CL_ENHANCE, 0xf000001f, 0x00000007,
  71         mmPA_SC_FORCE_EOV_MAX_CNTS, 0xffffffff, 0x00ffffff,
  72         mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
  73         mmPA_SC_MODE_CNTL_1, 0x07ffffff, 0x4e000000,
  74         mmPA_SC_RASTER_CONFIG, 0x3f3f3fff, 0x2a00126a,
  75         0x000c, 0xffffffff, 0x0040,
  76         0x000d, 0x00000040, 0x00004040,
  77         mmSPI_CONFIG_CNTL, 0x07ffffff, 0x03000000,
  78         mmSQ_DED_CNT, 0x01ff1f3f, 0x00000000,
  79         mmSQ_SEC_CNT, 0x01ff1f3f, 0x00000000,
  80         mmSX_DEBUG_1, 0x0000007f, 0x00000020,
  81         mmTA_CNTL_AUX, 0x00010000, 0x00010000,
  82         mmTCP_ADDR_CONFIG, 0x00000200, 0x000002fb,
  83         mmTCP_CHAN_STEER_HI, 0xffffffff, 0x0000543b,
  84         mmTCP_CHAN_STEER_LO, 0xffffffff, 0xa9210876,
  85         mmVGT_FIFO_DEPTHS, 0xffffffff, 0x000fff40,
  86         mmVGT_GS_VERTEX_REUSE, 0x0000001f, 0x00000010,
  87         mmVM_CONTEXT0_CNTL, 0x20000000, 0x20fffed8,
  88         mmVM_L2_CG, 0x000c0fc0, 0x000c0400,
  89         mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0xffffffff,
  90         mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
  91         mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
  92         mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
  93 };
  94
  95 static const u32 tahiti_golden_registers2[] =
  96 {
  97         mmMCIF_MEM_CONTROL, 0x00000001, 0x00000001,
  98 };
  99
 100 static const u32 tahiti_golden_rlc_registers[] =
 101 {
 102         mmGB_ADDR_CONFIG, 0xffffffff, 0x12011003,
 103         mmRLC_LB_PARAMS, 0xffffffff, 0x00601005,
 104         0x311f, 0xffffffff, 0x10104040,
 105         0x3122, 0xffffffff, 0x0100000a,
 106         mmRLC_LB_CNTR_MAX, 0xffffffff, 0x00000800,
 107         mmRLC_LB_CNTL, 0xffffffff, 0x800000f4,
 108         mmUVD_CGC_GATE, 0x00000008, 0x00000000,
 109 };
 110
 111 static const u32 pitcairn_golden_registers[] =
 112 {
 113         mmAZALIA_SCLK_CONTROL, 0x00000030, 0x00000011,
 114         mmCB_HW_CONTROL, 0x00010000, 0x00018208,
 115         mmDB_DEBUG, 0xffffffff, 0x00000000,
 116         mmDB_DEBUG2, 0xf00fffff, 0x00000400,
 117         mmDB_DEBUG3, 0x0002021c, 0x00020200,
 118         mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
 119         0x340c, 0x000300c0, 0x00800040,
 120         0x360c, 0x000300c0, 0x00800040,
 121         mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
 122         mmFBC_MISC, 0x00200000, 0x50100000,
 123         mmDIG0_HDMI_CONTROL, 0x31000311, 0x00000011,
 124         mmMC_SEQ_PMG_PG_HWCNTL, 0x00073ffe, 0x000022a2,
 125         mmMC_XPB_P2P_BAR_CFG, 0x000007ff, 0x00000000,
 126         mmPA_CL_ENHANCE, 0xf000001f, 0x00000007,
 127         mmPA_SC_FORCE_EOV_MAX_CNTS, 0xffffffff, 0x00ffffff,
 128         mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
 129         mmPA_SC_MODE_CNTL_1, 0x07ffffff, 0x4e000000,
 130         mmPA_SC_RASTER_CONFIG, 0x3f3f3fff, 0x2a00126a,
 131         0x000c, 0xffffffff, 0x0040,
 132         0x000d, 0x00000040, 0x00004040,
 133         mmSPI_CONFIG_CNTL, 0x07ffffff, 0x03000000,
 134         mmSX_DEBUG_1, 0x0000007f, 0x00000020,
 135         mmTA_CNTL_AUX, 0x00010000, 0x00010000,
 136         mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f7,
 137         mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,
 138         mmTCP_CHAN_STEER_LO, 0xffffffff, 0x32761054,
 139         mmVGT_GS_VERTEX_REUSE, 0x0000001f, 0x00000010,
 140         mmVM_L2_CG, 0x000c0fc0, 0x000c0400,
 141         mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0xffffffff,
 142         mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 143         mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 144         mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 145 };
 146
 147 static const u32 pitcairn_golden_rlc_registers[] =
 148 {
 149         mmGB_ADDR_CONFIG, 0xffffffff, 0x12011003,
 150         mmRLC_LB_PARAMS, 0xffffffff, 0x00601004,
 151         0x311f, 0xffffffff, 0x10102020,
 152         0x3122, 0xffffffff, 0x01000020,
 153         mmRLC_LB_CNTR_MAX, 0xffffffff, 0x00000800,
 154         mmRLC_LB_CNTL, 0xffffffff, 0x800000a4,
 155 };
 156
 157 static const u32 verde_pg_init[] =
 158 {
 159         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x40000,
 160         mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x200010ff,
 161         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 162         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 163         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 164         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 165         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 166         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x7007,
 167         mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x300010ff,
 168         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 169         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 170         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 171         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 172         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 173         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x400000,
 174         mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x100010ff,
 175         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 176         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 177         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 178         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 179         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 180         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x120200,
 181         mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x500010ff,
 182         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 183         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 184         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 185         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 186         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 187         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x1e1e16,
 188         mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x600010ff,
 189         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 190         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 191         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 192         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 193         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 194         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x171f1e,
 195         mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x700010ff,
 196         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 197         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 198         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 199         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 200         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 201         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 202         mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x9ff,
 203         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x0,
 204         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x10000800,
 205         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xf,
 206         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xf,
 207         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x4,
 208         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1000051e,
 209         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xffff,
 210         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xffff,
 211         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x8,
 212         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x80500,
 213         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x12,
 214         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x9050c,
 215         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x1d,
 216         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xb052c,
 217         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x2a,
 218         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1053e,
 219         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x2d,
 220         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x10546,
 221         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x30,
 222         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xa054e,
 223         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x3c,
 224         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1055f,
 225         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x3f,
 226         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x10567,
 227         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x42,
 228         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1056f,
 229         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x45,
 230         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x10572,
 231         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x48,
 232         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x20575,
 233         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x4c,
 234         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x190801,
 235         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x67,
 236         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1082a,
 237         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x6a,
 238         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1b082d,
 239         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x87,
 240         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x310851,
 241         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xba,
 242         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x891,
 243         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xbc,
 244         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x893,
 245         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xbe,
 246         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x20895,
 247         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xc2,
 248         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x20899,
 249         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xc6,
 250         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x2089d,
 251         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xca,
 252         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x8a1,
 253         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xcc,
 254         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x8a3,
 255         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xce,
 256         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x308a5,
 257         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xd3,
 258         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x6d08cd,
 259         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x142,
 260         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x2000095a,
 261         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1,
 262         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x144,
 263         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x301f095b,
 264         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x165,
 265         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xc094d,
 266         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x173,
 267         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xf096d,
 268         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x184,
 269         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x15097f,
 270         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x19b,
 271         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xc0998,
 272         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x1a9,
 273         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x409a7,
 274         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x1af,
 275         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xcdc,
 276         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x1b1,
 277         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x800,
 278         mmGMCON_RENG_EXECUTE, 0xffffffff, 0x6c9b2000,
 279         mmGMCON_MISC2, 0xfc00, 0x2000,
 280         mmGMCON_MISC3, 0xffffffff, 0xfc0,
 281         mmMC_PMG_AUTO_CFG, 0x00000100, 0x100,
 282 };
 283
 284 static const u32 verde_golden_rlc_registers[] =
 285 {
 286         mmGB_ADDR_CONFIG, 0xffffffff, 0x02010002,
 287         mmRLC_LB_PARAMS, 0xffffffff, 0x033f1005,
 288         0x311f, 0xffffffff, 0x10808020,
 289         0x3122, 0xffffffff, 0x00800008,
 290         mmRLC_LB_CNTR_MAX, 0xffffffff, 0x00001000,
 291         mmRLC_LB_CNTL, 0xffffffff, 0x80010014,
 292 };
 293
 294 static const u32 verde_golden_registers[] =
 295 {
 296         mmAZALIA_SCLK_CONTROL, 0x00000030, 0x00000011,
 297         mmCB_HW_CONTROL, 0x00010000, 0x00018208,
 298         mmDB_DEBUG, 0xffffffff, 0x00000000,
 299         mmDB_DEBUG2, 0xf00fffff, 0x00000400,
 300         mmDB_DEBUG3, 0x0002021c, 0x00020200,
 301         mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
 302         0x340c, 0x000300c0, 0x00800040,
 303         0x360c, 0x000300c0, 0x00800040,
 304         mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
 305         mmFBC_MISC, 0x00200000, 0x50100000,
 306         mmDIG0_HDMI_CONTROL, 0x31000311, 0x00000011,
 307         mmMC_SEQ_PMG_PG_HWCNTL, 0x00073ffe, 0x000022a2,
 308         mmMC_XPB_P2P_BAR_CFG, 0x000007ff, 0x00000000,
 309         mmPA_CL_ENHANCE, 0xf000001f, 0x00000007,
 310         mmPA_SC_FORCE_EOV_MAX_CNTS, 0xffffffff, 0x00ffffff,
 311         mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
 312         mmPA_SC_MODE_CNTL_1, 0x07ffffff, 0x4e000000,
 313         mmPA_SC_RASTER_CONFIG, 0x3f3f3fff, 0x0000124a,
 314         0x000c, 0xffffffff, 0x0040,
 315         0x000d, 0x00000040, 0x00004040,
 316         mmSPI_CONFIG_CNTL, 0x07ffffff, 0x03000000,
 317         mmSQ_DED_CNT, 0x01ff1f3f, 0x00000000,
 318         mmSQ_SEC_CNT, 0x01ff1f3f, 0x00000000,
 319         mmSX_DEBUG_1, 0x0000007f, 0x00000020,
 320         mmTA_CNTL_AUX, 0x00010000, 0x00010000,
 321         mmTCP_ADDR_CONFIG, 0x000003ff, 0x00000003,
 322         mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,
 323         mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00001032,
 324         mmVGT_GS_VERTEX_REUSE, 0x0000001f, 0x00000010,
 325         mmVM_L2_CG, 0x000c0fc0, 0x000c0400,
 326         mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0xffffffff,
 327         mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 328         mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 329         mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 330 };
 331
 332 static const u32 oland_golden_registers[] =
 333 {
 334         mmAZALIA_SCLK_CONTROL, 0x00000030, 0x00000011,
 335         mmCB_HW_CONTROL, 0x00010000, 0x00018208,
 336         mmDB_DEBUG, 0xffffffff, 0x00000000,
 337         mmDB_DEBUG2, 0xf00fffff, 0x00000400,
 338         mmDB_DEBUG3, 0x0002021c, 0x00020200,
 339         mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
 340         0x340c, 0x000300c0, 0x00800040,
 341         0x360c, 0x000300c0, 0x00800040,
 342         mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
 343         mmFBC_MISC, 0x00200000, 0x50100000,
 344         mmDIG0_HDMI_CONTROL, 0x31000311, 0x00000011,
 345         mmMC_SEQ_PMG_PG_HWCNTL, 0x00073ffe, 0x000022a2,
 346         mmMC_XPB_P2P_BAR_CFG, 0x000007ff, 0x00000000,
 347         mmPA_CL_ENHANCE, 0xf000001f, 0x00000007,
 348         mmPA_SC_FORCE_EOV_MAX_CNTS, 0xffffffff, 0x00ffffff,
 349         mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
 350         mmPA_SC_MODE_CNTL_1, 0x07ffffff, 0x4e000000,
 351         mmPA_SC_RASTER_CONFIG, 0x3f3f3fff, 0x00000082,
 352         0x000c, 0xffffffff, 0x0040,
 353         0x000d, 0x00000040, 0x00004040,
 354         mmSPI_CONFIG_CNTL, 0x07ffffff, 0x03000000,
 355         mmSX_DEBUG_1, 0x0000007f, 0x00000020,
 356         mmTA_CNTL_AUX, 0x00010000, 0x00010000,
 357         mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f3,
 358         mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,
 359         mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003210,
 360         mmVGT_GS_VERTEX_REUSE, 0x0000001f, 0x00000010,
 361         mmVM_L2_CG, 0x000c0fc0, 0x000c0400,
 362         mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0xffffffff,
 363         mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 364         mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 365         mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 366
 367 };
 368
 369 static const u32 oland_golden_rlc_registers[] =
 370 {
 371         mmGB_ADDR_CONFIG, 0xffffffff, 0x02010002,
 372         mmRLC_LB_PARAMS, 0xffffffff, 0x00601005,
 373         0x311f, 0xffffffff, 0x10104040,
 374         0x3122, 0xffffffff, 0x0100000a,
 375         mmRLC_LB_CNTR_MAX, 0xffffffff, 0x00000800,
 376         mmRLC_LB_CNTL, 0xffffffff, 0x800000f4,
 377 };
 378
 379 static const u32 hainan_golden_registers[] =
 380 {
 381         0x17bc, 0x00000030, 0x00000011,
 382         mmCB_HW_CONTROL, 0x00010000, 0x00018208,
 383         mmDB_DEBUG, 0xffffffff, 0x00000000,
 384         mmDB_DEBUG2, 0xf00fffff, 0x00000400,
 385         mmDB_DEBUG3, 0x0002021c, 0x00020200,
 386         0x031e, 0x00000080, 0x00000000,
 387         0x3430, 0xff000fff, 0x00000100,
 388         0x340c, 0x000300c0, 0x00800040,
 389         0x3630, 0xff000fff, 0x00000100,
 390         0x360c, 0x000300c0, 0x00800040,
 391         0x16ec, 0x000000f0, 0x00000070,
 392         0x16f0, 0x00200000, 0x50100000,
 393         0x1c0c, 0x31000311, 0x00000011,
 394         mmMC_SEQ_PMG_PG_HWCNTL, 0x00073ffe, 0x000022a2,
 395         mmMC_XPB_P2P_BAR_CFG, 0x000007ff, 0x00000000,
 396         mmPA_CL_ENHANCE, 0xf000001f, 0x00000007,
 397         mmPA_SC_FORCE_EOV_MAX_CNTS, 0xffffffff, 0x00ffffff,
 398         mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
 399         mmPA_SC_MODE_CNTL_1, 0x07ffffff, 0x4e000000,
 400         mmPA_SC_RASTER_CONFIG, 0x3f3f3fff, 0x00000000,
 401         0x000c, 0xffffffff, 0x0040,
 402         0x000d, 0x00000040, 0x00004040,
 403         mmSPI_CONFIG_CNTL, 0x03e00000, 0x03600000,
 404         mmSX_DEBUG_1, 0x0000007f, 0x00000020,
 405         mmTA_CNTL_AUX, 0x00010000, 0x00010000,
 406         mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f1,
 407         mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,
 408         mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003210,
 409         mmVGT_GS_VERTEX_REUSE, 0x0000001f, 0x00000010,
 410         mmVM_L2_CG, 0x000c0fc0, 0x000c0400,
 411         mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0xffffffff,
 412         mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 413         mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 414         mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 415 };
 416
 417 static const u32 hainan_golden_registers2[] =
 418 {
 419         mmGB_ADDR_CONFIG, 0xffffffff, 0x2011003,
 420 };
 421
 422 static const u32 tahiti_mgcg_cgcg_init[] =
 423 {
 424         mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xfffffffc,
 425         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 426         mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 427         mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
 428         mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
 429         mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
 430         mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
 431         mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
 432         mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
 433         mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
 434         mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
 435         mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
 436         mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
 437         mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
 438         mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
 439         mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
 440         mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
 441         mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
 442         mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
 443         mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
 444         mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
 445         mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
 446         mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
 447         mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 448         mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 449         mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
 450         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 451         0x2458, 0xffffffff, 0x00010000,
 452         0x2459, 0xffffffff, 0x00030002,
 453         0x245a, 0xffffffff, 0x00040007,
 454         0x245b, 0xffffffff, 0x00060005,
 455         0x245c, 0xffffffff, 0x00090008,
 456         0x245d, 0xffffffff, 0x00020001,
 457         0x245e, 0xffffffff, 0x00040003,
 458         0x245f, 0xffffffff, 0x00000007,
 459         0x2460, 0xffffffff, 0x00060005,
 460         0x2461, 0xffffffff, 0x00090008,
 461         0x2462, 0xffffffff, 0x00030002,
 462         0x2463, 0xffffffff, 0x00050004,
 463         0x2464, 0xffffffff, 0x00000008,
 464         0x2465, 0xffffffff, 0x00070006,
 465         0x2466, 0xffffffff, 0x000a0009,
 466         0x2467, 0xffffffff, 0x00040003,
 467         0x2468, 0xffffffff, 0x00060005,
 468         0x2469, 0xffffffff, 0x00000009,
 469         0x246a, 0xffffffff, 0x00080007,
 470         0x246b, 0xffffffff, 0x000b000a,
 471         0x246c, 0xffffffff, 0x00050004,
 472         0x246d, 0xffffffff, 0x00070006,
 473         0x246e, 0xffffffff, 0x0008000b,
 474         0x246f, 0xffffffff, 0x000a0009,
 475         0x2470, 0xffffffff, 0x000d000c,
 476         0x2471, 0xffffffff, 0x00060005,
 477         0x2472, 0xffffffff, 0x00080007,
 478         0x2473, 0xffffffff, 0x0000000b,
 479         0x2474, 0xffffffff, 0x000a0009,
 480         0x2475, 0xffffffff, 0x000d000c,
 481         0x2476, 0xffffffff, 0x00070006,
 482         0x2477, 0xffffffff, 0x00090008,
 483         0x2478, 0xffffffff, 0x0000000c,
 484         0x2479, 0xffffffff, 0x000b000a,
 485         0x247a, 0xffffffff, 0x000e000d,
 486         0x247b, 0xffffffff, 0x00080007,
 487         0x247c, 0xffffffff, 0x000a0009,
 488         0x247d, 0xffffffff, 0x0000000d,
 489         0x247e, 0xffffffff, 0x000c000b,
 490         0x247f, 0xffffffff, 0x000f000e,
 491         0x2480, 0xffffffff, 0x00090008,
 492         0x2481, 0xffffffff, 0x000b000a,
 493         0x2482, 0xffffffff, 0x000c000f,
 494         0x2483, 0xffffffff, 0x000e000d,
 495         0x2484, 0xffffffff, 0x00110010,
 496         0x2485, 0xffffffff, 0x000a0009,
 497         0x2486, 0xffffffff, 0x000c000b,
 498         0x2487, 0xffffffff, 0x0000000f,
 499         0x2488, 0xffffffff, 0x000e000d,
 500         0x2489, 0xffffffff, 0x00110010,
 501         0x248a, 0xffffffff, 0x000b000a,
 502         0x248b, 0xffffffff, 0x000d000c,
 503         0x248c, 0xffffffff, 0x00000010,
 504         0x248d, 0xffffffff, 0x000f000e,
 505         0x248e, 0xffffffff, 0x00120011,
 506         0x248f, 0xffffffff, 0x000c000b,
 507         0x2490, 0xffffffff, 0x000e000d,
 508         0x2491, 0xffffffff, 0x00000011,
 509         0x2492, 0xffffffff, 0x0010000f,
 510         0x2493, 0xffffffff, 0x00130012,
 511         0x2494, 0xffffffff, 0x000d000c,
 512         0x2495, 0xffffffff, 0x000f000e,
 513         0x2496, 0xffffffff, 0x00100013,
 514         0x2497, 0xffffffff, 0x00120011,
 515         0x2498, 0xffffffff, 0x00150014,
 516         0x2499, 0xffffffff, 0x000e000d,
 517         0x249a, 0xffffffff, 0x0010000f,
 518         0x249b, 0xffffffff, 0x00000013,
 519         0x249c, 0xffffffff, 0x00120011,
 520         0x249d, 0xffffffff, 0x00150014,
 521         0x249e, 0xffffffff, 0x000f000e,
 522         0x249f, 0xffffffff, 0x00110010,
 523         0x24a0, 0xffffffff, 0x00000014,
 524         0x24a1, 0xffffffff, 0x00130012,
 525         0x24a2, 0xffffffff, 0x00160015,
 526         0x24a3, 0xffffffff, 0x0010000f,
 527         0x24a4, 0xffffffff, 0x00120011,
 528         0x24a5, 0xffffffff, 0x00000015,
 529         0x24a6, 0xffffffff, 0x00140013,
 530         0x24a7, 0xffffffff, 0x00170016,
 531         mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,
 532         mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
 533         mmRLC_GCPM_GENERAL_3, 0xffffffff, 0x00000080,
 534         mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,
 535         0x000c, 0xffffffff, 0x0000001c,
 536         0x000d, 0x000f0000, 0x000f0000,
 537         0x0583, 0xffffffff, 0x00000100,
 538         mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,
 539         mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,
 540         mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
 541         mmMC_CITF_MISC_WR_CG, 0x000c0000, 0x000c0000,
 542         mmMC_CITF_MISC_RD_CG, 0x000c0000, 0x000c0000,
 543         mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
 544         0x157a, 0x00000001, 0x00000001,
 545         mmHDP_MEM_POWER_LS, 0x00000001, 0x00000001,
 546         mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
 547         mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
 548         0x3430, 0xfffffff0, 0x00000100,
 549         0x3630, 0xfffffff0, 0x00000100,
 550 };
 551 static const u32 pitcairn_mgcg_cgcg_init[] =
 552 {
 553         mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xfffffffc,
 554         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 555         mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 556         mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
 557         mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
 558         mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
 559         mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
 560         mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
 561         mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
 562         mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
 563         mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
 564         mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
 565         mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
 566         mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
 567         mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
 568         mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
 569         mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
 570         mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
 571         mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
 572         mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
 573         mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
 574         mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
 575         mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
 576         mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 577         mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 578         mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
 579         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 580         0x2458, 0xffffffff, 0x00010000,
 581         0x2459, 0xffffffff, 0x00030002,
 582         0x245a, 0xffffffff, 0x00040007,
 583         0x245b, 0xffffffff, 0x00060005,
 584         0x245c, 0xffffffff, 0x00090008,
 585         0x245d, 0xffffffff, 0x00020001,
 586         0x245e, 0xffffffff, 0x00040003,
 587         0x245f, 0xffffffff, 0x00000007,
 588         0x2460, 0xffffffff, 0x00060005,
 589         0x2461, 0xffffffff, 0x00090008,
 590         0x2462, 0xffffffff, 0x00030002,
 591         0x2463, 0xffffffff, 0x00050004,
 592         0x2464, 0xffffffff, 0x00000008,
 593         0x2465, 0xffffffff, 0x00070006,
 594         0x2466, 0xffffffff, 0x000a0009,
 595         0x2467, 0xffffffff, 0x00040003,
 596         0x2468, 0xffffffff, 0x00060005,
 597         0x2469, 0xffffffff, 0x00000009,
 598         0x246a, 0xffffffff, 0x00080007,
 599         0x246b, 0xffffffff, 0x000b000a,
 600         0x246c, 0xffffffff, 0x00050004,
 601         0x246d, 0xffffffff, 0x00070006,
 602         0x246e, 0xffffffff, 0x0008000b,
 603         0x246f, 0xffffffff, 0x000a0009,
 604         0x2470, 0xffffffff, 0x000d000c,
 605         0x2480, 0xffffffff, 0x00090008,
 606         0x2481, 0xffffffff, 0x000b000a,
 607         0x2482, 0xffffffff, 0x000c000f,
 608         0x2483, 0xffffffff, 0x000e000d,
 609         0x2484, 0xffffffff, 0x00110010,
 610         0x2485, 0xffffffff, 0x000a0009,
 611         0x2486, 0xffffffff, 0x000c000b,
 612         0x2487, 0xffffffff, 0x0000000f,
 613         0x2488, 0xffffffff, 0x000e000d,
 614         0x2489, 0xffffffff, 0x00110010,
 615         0x248a, 0xffffffff, 0x000b000a,
 616         0x248b, 0xffffffff, 0x000d000c,
 617         0x248c, 0xffffffff, 0x00000010,
 618         0x248d, 0xffffffff, 0x000f000e,
 619         0x248e, 0xffffffff, 0x00120011,
 620         0x248f, 0xffffffff, 0x000c000b,
 621         0x2490, 0xffffffff, 0x000e000d,
 622         0x2491, 0xffffffff, 0x00000011,
 623         0x2492, 0xffffffff, 0x0010000f,
 624         0x2493, 0xffffffff, 0x00130012,
 625         0x2494, 0xffffffff, 0x000d000c,
 626         0x2495, 0xffffffff, 0x000f000e,
 627         0x2496, 0xffffffff, 0x00100013,
 628         0x2497, 0xffffffff, 0x00120011,
 629         0x2498, 0xffffffff, 0x00150014,
 630         mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,
 631         mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
 632         mmRLC_GCPM_GENERAL_3, 0xffffffff, 0x00000080,
 633         mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,
 634         0x000c, 0xffffffff, 0x0000001c,
 635         0x000d, 0x000f0000, 0x000f0000,
 636         0x0583, 0xffffffff, 0x00000100,
 637         mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,
 638         mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,
 639         mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
 640         mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
 641         0x157a, 0x00000001, 0x00000001,
 642         mmHDP_MEM_POWER_LS, 0x00000001, 0x00000001,
 643         mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
 644         mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
 645         0x3430, 0xfffffff0, 0x00000100,
 646         0x3630, 0xfffffff0, 0x00000100,
 647 };
 648
 649 static const u32 verde_mgcg_cgcg_init[] =
 650 {
 651         mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xfffffffc,
 652         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 653         mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 654         mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
 655         mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
 656         mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
 657         mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
 658         mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
 659         mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
 660         mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
 661         mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
 662         mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
 663         mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
 664         mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
 665         mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
 666         mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
 667         mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
 668         mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
 669         mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
 670         mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
 671         mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
 672         mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
 673         mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
 674         mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 675         mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 676         mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
 677         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 678         0x2458, 0xffffffff, 0x00010000,
 679         0x2459, 0xffffffff, 0x00030002,
 680         0x245a, 0xffffffff, 0x00040007,
 681         0x245b, 0xffffffff, 0x00060005,
 682         0x245c, 0xffffffff, 0x00090008,
 683         0x245d, 0xffffffff, 0x00020001,
 684         0x245e, 0xffffffff, 0x00040003,
 685         0x245f, 0xffffffff, 0x00000007,
 686         0x2460, 0xffffffff, 0x00060005,
 687         0x2461, 0xffffffff, 0x00090008,
 688         0x2462, 0xffffffff, 0x00030002,
 689         0x2463, 0xffffffff, 0x00050004,
 690         0x2464, 0xffffffff, 0x00000008,
 691         0x2465, 0xffffffff, 0x00070006,
 692         0x2466, 0xffffffff, 0x000a0009,
 693         0x2467, 0xffffffff, 0x00040003,
 694         0x2468, 0xffffffff, 0x00060005,
 695         0x2469, 0xffffffff, 0x00000009,
 696         0x246a, 0xffffffff, 0x00080007,
 697         0x246b, 0xffffffff, 0x000b000a,
 698         0x246c, 0xffffffff, 0x00050004,
 699         0x246d, 0xffffffff, 0x00070006,
 700         0x246e, 0xffffffff, 0x0008000b,
 701         0x246f, 0xffffffff, 0x000a0009,
 702         0x2470, 0xffffffff, 0x000d000c,
 703         0x2480, 0xffffffff, 0x00090008,
 704         0x2481, 0xffffffff, 0x000b000a,
 705         0x2482, 0xffffffff, 0x000c000f,
 706         0x2483, 0xffffffff, 0x000e000d,
 707         0x2484, 0xffffffff, 0x00110010,
 708         0x2485, 0xffffffff, 0x000a0009,
 709         0x2486, 0xffffffff, 0x000c000b,
 710         0x2487, 0xffffffff, 0x0000000f,
 711         0x2488, 0xffffffff, 0x000e000d,
 712         0x2489, 0xffffffff, 0x00110010,
 713         0x248a, 0xffffffff, 0x000b000a,
 714         0x248b, 0xffffffff, 0x000d000c,
 715         0x248c, 0xffffffff, 0x00000010,
 716         0x248d, 0xffffffff, 0x000f000e,
 717         0x248e, 0xffffffff, 0x00120011,
 718         0x248f, 0xffffffff, 0x000c000b,
 719         0x2490, 0xffffffff, 0x000e000d,
 720         0x2491, 0xffffffff, 0x00000011,
 721         0x2492, 0xffffffff, 0x0010000f,
 722         0x2493, 0xffffffff, 0x00130012,
 723         0x2494, 0xffffffff, 0x000d000c,
 724         0x2495, 0xffffffff, 0x000f000e,
 725         0x2496, 0xffffffff, 0x00100013,
 726         0x2497, 0xffffffff, 0x00120011,
 727         0x2498, 0xffffffff, 0x00150014,
 728         mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,
 729         mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
 730         mmRLC_GCPM_GENERAL_3, 0xffffffff, 0x00000080,
 731         mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,
 732         0x000c, 0xffffffff, 0x0000001c,
 733         0x000d, 0x000f0000, 0x000f0000,
 734         0x0583, 0xffffffff, 0x00000100,
 735         mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,
 736         mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,
 737         mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
 738         mmMC_CITF_MISC_WR_CG, 0x000c0000, 0x000c0000,
 739         mmMC_CITF_MISC_RD_CG, 0x000c0000, 0x000c0000,
 740         mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
 741         0x157a, 0x00000001, 0x00000001,
 742         mmHDP_MEM_POWER_LS, 0x00000001, 0x00000001,
 743         mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
 744         mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
 745         0x3430, 0xfffffff0, 0x00000100,
 746         0x3630, 0xfffffff0, 0x00000100,
 747 };
 748
 749 static const u32 oland_mgcg_cgcg_init[] =
 750 {
 751         mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xfffffffc,
 752         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 753         mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 754         mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
 755         mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
 756         mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
 757         mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
 758         mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
 759         mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
 760         mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
 761         mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
 762         mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
 763         mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
 764         mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
 765         mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
 766         mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
 767         mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
 768         mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
 769         mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
 770         mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
 771         mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
 772         mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
 773         mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
 774         mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 775         mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 776         mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
 777         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 778         0x2458, 0xffffffff, 0x00010000,
 779         0x2459, 0xffffffff, 0x00030002,
 780         0x245a, 0xffffffff, 0x00040007,
 781         0x245b, 0xffffffff, 0x00060005,
 782         0x245c, 0xffffffff, 0x00090008,
 783         0x245d, 0xffffffff, 0x00020001,
 784         0x245e, 0xffffffff, 0x00040003,
 785         0x245f, 0xffffffff, 0x00000007,
 786         0x2460, 0xffffffff, 0x00060005,
 787         0x2461, 0xffffffff, 0x00090008,
 788         0x2462, 0xffffffff, 0x00030002,
 789         0x2463, 0xffffffff, 0x00050004,
 790         0x2464, 0xffffffff, 0x00000008,
 791         0x2465, 0xffffffff, 0x00070006,
 792         0x2466, 0xffffffff, 0x000a0009,
 793         0x2467, 0xffffffff, 0x00040003,
 794         0x2468, 0xffffffff, 0x00060005,
 795         0x2469, 0xffffffff, 0x00000009,
 796         0x246a, 0xffffffff, 0x00080007,
 797         0x246b, 0xffffffff, 0x000b000a,
 798         0x246c, 0xffffffff, 0x00050004,
 799         0x246d, 0xffffffff, 0x00070006,
 800         0x246e, 0xffffffff, 0x0008000b,
 801         0x246f, 0xffffffff, 0x000a0009,
 802         0x2470, 0xffffffff, 0x000d000c,
 803         0x2471, 0xffffffff, 0x00060005,
 804         0x2472, 0xffffffff, 0x00080007,
 805         0x2473, 0xffffffff, 0x0000000b,
 806         0x2474, 0xffffffff, 0x000a0009,
 807         0x2475, 0xffffffff, 0x000d000c,
 808         mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,
 809         mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
 810         mmRLC_GCPM_GENERAL_3, 0xffffffff, 0x00000080,
 811         mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,
 812         0x000c, 0xffffffff, 0x0000001c,
 813         0x000d, 0x000f0000, 0x000f0000,
 814         0x0583, 0xffffffff, 0x00000100,
 815         mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,
 816         mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,
 817         mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
 818         mmMC_CITF_MISC_WR_CG, 0x000c0000, 0x000c0000,
 819         mmMC_CITF_MISC_RD_CG, 0x000c0000, 0x000c0000,
 820         mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
 821         0x157a, 0x00000001, 0x00000001,
 822         mmHDP_MEM_POWER_LS, 0x00000001, 0x00000001,
 823         mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
 824         mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
 825         0x3430, 0xfffffff0, 0x00000100,
 826         0x3630, 0xfffffff0, 0x00000100,
 827 };
 828
 829 static const u32 hainan_mgcg_cgcg_init[] =
 830 {
 831         mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xfffffffc,
 832         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 833         mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 834         mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
 835         mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
 836         mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
 837         mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
 838         mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
 839         mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
 840         mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
 841         mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
 842         mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
 843         mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
 844         mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
 845         mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
 846         mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
 847         mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
 848         mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
 849         mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
 850         mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
 851         mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
 852         mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
 853         mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
 854         mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 855         mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 856         mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
 857         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 858         0x2458, 0xffffffff, 0x00010000,
 859         0x2459, 0xffffffff, 0x00030002,
 860         0x245a, 0xffffffff, 0x00040007,
 861         0x245b, 0xffffffff, 0x00060005,
 862         0x245c, 0xffffffff, 0x00090008,
 863         0x245d, 0xffffffff, 0x00020001,
 864         0x245e, 0xffffffff, 0x00040003,
 865         0x245f, 0xffffffff, 0x00000007,
 866         0x2460, 0xffffffff, 0x00060005,
 867         0x2461, 0xffffffff, 0x00090008,
 868         0x2462, 0xffffffff, 0x00030002,
 869         0x2463, 0xffffffff, 0x00050004,
 870         0x2464, 0xffffffff, 0x00000008,
 871         0x2465, 0xffffffff, 0x00070006,
 872         0x2466, 0xffffffff, 0x000a0009,
 873         0x2467, 0xffffffff, 0x00040003,
 874         0x2468, 0xffffffff, 0x00060005,
 875         0x2469, 0xffffffff, 0x00000009,
 876         0x246a, 0xffffffff, 0x00080007,
 877         0x246b, 0xffffffff, 0x000b000a,
 878         0x246c, 0xffffffff, 0x00050004,
 879         0x246d, 0xffffffff, 0x00070006,
 880         0x246e, 0xffffffff, 0x0008000b,
 881         0x246f, 0xffffffff, 0x000a0009,
 882         0x2470, 0xffffffff, 0x000d000c,
 883         0x2471, 0xffffffff, 0x00060005,
 884         0x2472, 0xffffffff, 0x00080007,
 885         0x2473, 0xffffffff, 0x0000000b,
 886         0x2474, 0xffffffff, 0x000a0009,
 887         0x2475, 0xffffffff, 0x000d000c,
 888         mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,
 889         mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
 890         mmRLC_GCPM_GENERAL_3, 0xffffffff, 0x00000080,
 891         mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,
 892         0x000c, 0xffffffff, 0x0000001c,
 893         0x000d, 0x000f0000, 0x000f0000,
 894         0x0583, 0xffffffff, 0x00000100,
 895         0x0409, 0xffffffff, 0x00000100,
 896         mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
 897         mmMC_CITF_MISC_WR_CG, 0x000c0000, 0x000c0000,
 898         mmMC_CITF_MISC_RD_CG, 0x000c0000, 0x000c0000,
 899         mmHDP_MEM_POWER_LS, 0x00000001, 0x00000001,
 900         mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
 901         mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
 902         0x3430, 0xfffffff0, 0x00000100,
 903         0x3630, 0xfffffff0, 0x00000100,
 904 };
 905
 906 static u32 si_pcie_rreg(struct amdgpu_device *adev, u32 reg)
 907 {
 908         unsigned long flags;
 909         u32 r;
 910
 911         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 912         WREG32(AMDGPU_PCIE_INDEX, reg);
 913         (void)RREG32(AMDGPU_PCIE_INDEX);
 914         r = RREG32(AMDGPU_PCIE_DATA);
 915         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 916         return r;
 917 }
 918
 919 static void si_pcie_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 920 {
 921         unsigned long flags;
 922
 923         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 924         WREG32(AMDGPU_PCIE_INDEX, reg);
 925         (void)RREG32(AMDGPU_PCIE_INDEX);
 926         WREG32(AMDGPU_PCIE_DATA, v);
 927         (void)RREG32(AMDGPU_PCIE_DATA);
 928         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 929 }
 930
 931 static u32 si_pciep_rreg(struct amdgpu_device *adev, u32 reg)
 932 {
 933         unsigned long flags;
 934         u32 r;
 935
 936         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 937         WREG32(PCIE_PORT_INDEX, ((reg) & 0xff));
 938         (void)RREG32(PCIE_PORT_INDEX);
 939         r = RREG32(PCIE_PORT_DATA);
 940         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 941         return r;
 942 }
 943
 944 static void si_pciep_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 945 {
 946         unsigned long flags;
 947
 948         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 949         WREG32(PCIE_PORT_INDEX, ((reg) & 0xff));
 950         (void)RREG32(PCIE_PORT_INDEX);
 951         WREG32(PCIE_PORT_DATA, (v));
 952         (void)RREG32(PCIE_PORT_DATA);
 953         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 954 }
 955
 956 static u32 si_smc_rreg(struct amdgpu_device *adev, u32 reg)
 957 {
 958         unsigned long flags;
 959         u32 r;
 960
 961         spin_lock_irqsave(&adev->smc_idx_lock, flags);
 962         WREG32(SMC_IND_INDEX_0, (reg));
 963         r = RREG32(SMC_IND_DATA_0);
 964         spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
 965         return r;
 966 }
 967
 968 static void si_smc_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 969 {
 970         unsigned long flags;
 971
 972         spin_lock_irqsave(&adev->smc_idx_lock, flags);
 973         WREG32(SMC_IND_INDEX_0, (reg));
 974         WREG32(SMC_IND_DATA_0, (v));
 975         spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
 976 }
 977
 978 static u32 si_uvd_ctx_rreg(struct amdgpu_device *adev, u32 reg)
 979 {
 980         unsigned long flags;
 981         u32 r;
 982
 983         spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
 984         WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
 985         r = RREG32(mmUVD_CTX_DATA);
 986         spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
 987         return r;
 988 }
 989
 990 static void si_uvd_ctx_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 991 {
 992         unsigned long flags;
 993
 994         spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
 995         WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
 996         WREG32(mmUVD_CTX_DATA, (v));
 997         spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
 998 }
 999
1000 static struct amdgpu_allowed_register_entry si_allowed_read_registers[] = {
1001         {GRBM_STATUS},
1002         {mmGRBM_STATUS2},
1003         {mmGRBM_STATUS_SE0},
1004         {mmGRBM_STATUS_SE1},
1005         {mmSRBM_STATUS},
1006         {mmSRBM_STATUS2},
1007         {DMA_STATUS_REG + DMA0_REGISTER_OFFSET},
1008         {DMA_STATUS_REG + DMA1_REGISTER_OFFSET},
1009         {mmCP_STAT},
1010         {mmCP_STALLED_STAT1},
1011         {mmCP_STALLED_STAT2},
1012         {mmCP_STALLED_STAT3},
1013         {GB_ADDR_CONFIG},
1014         {MC_ARB_RAMCFG},
1015         {GB_TILE_MODE0},
1016         {GB_TILE_MODE1},
1017         {GB_TILE_MODE2},
1018         {GB_TILE_MODE3},
1019         {GB_TILE_MODE4},
1020         {GB_TILE_MODE5},
1021         {GB_TILE_MODE6},
1022         {GB_TILE_MODE7},
1023         {GB_TILE_MODE8},
1024         {GB_TILE_MODE9},
1025         {GB_TILE_MODE10},
1026         {GB_TILE_MODE11},
1027         {GB_TILE_MODE12},
1028         {GB_TILE_MODE13},
1029         {GB_TILE_MODE14},
1030         {GB_TILE_MODE15},
1031         {GB_TILE_MODE16},
1032         {GB_TILE_MODE17},
1033         {GB_TILE_MODE18},
1034         {GB_TILE_MODE19},
1035         {GB_TILE_MODE20},
1036         {GB_TILE_MODE21},
1037         {GB_TILE_MODE22},
1038         {GB_TILE_MODE23},
1039         {GB_TILE_MODE24},
1040         {GB_TILE_MODE25},
1041         {GB_TILE_MODE26},
1042         {GB_TILE_MODE27},
1043         {GB_TILE_MODE28},
1044         {GB_TILE_MODE29},
1045         {GB_TILE_MODE30},
1046         {GB_TILE_MODE31},
1047         {CC_RB_BACKEND_DISABLE, true},
1048         {GC_USER_RB_BACKEND_DISABLE, true},
1049         {PA_SC_RASTER_CONFIG, true},
1050 };
1051
1052 static uint32_t si_get_register_value(struct amdgpu_device *adev,
1053                                       bool indexed, u32 se_num,
1054                                       u32 sh_num, u32 reg_offset)
1055 {
1056         if (indexed) {
1057                 uint32_t val;
1058                 unsigned se_idx = (se_num == 0xffffffff) ? 0 : se_num;
1059                 unsigned sh_idx = (sh_num == 0xffffffff) ? 0 : sh_num;
1060
1061                 switch (reg_offset) {
1062                 case mmCC_RB_BACKEND_DISABLE:
1063                         return adev->gfx.config.rb_config[se_idx][sh_idx].rb_backend_disable;
1064                 case mmGC_USER_RB_BACKEND_DISABLE:
1065                         return adev->gfx.config.rb_config[se_idx][sh_idx].user_rb_backend_disable;
1066                 case mmPA_SC_RASTER_CONFIG:
1067                         return adev->gfx.config.rb_config[se_idx][sh_idx].raster_config;
1068                 }
1069
1070                 mutex_lock(&adev->grbm_idx_mutex);
1071                 if (se_num != 0xffffffff || sh_num != 0xffffffff)
1072                         amdgpu_gfx_select_se_sh(adev, se_num, sh_num, 0xffffffff);
1073
1074                 val = RREG32(reg_offset);
1075
1076                 if (se_num != 0xffffffff || sh_num != 0xffffffff)
1077                         amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
1078                 mutex_unlock(&adev->grbm_idx_mutex);
1079                 return val;
1080         } else {
1081                 unsigned idx;
1082
1083                 switch (reg_offset) {
1084                 case mmGB_ADDR_CONFIG:
1085                         return adev->gfx.config.gb_addr_config;
1086                 case mmMC_ARB_RAMCFG:
1087                         return adev->gfx.config.mc_arb_ramcfg;
1088                 case mmGB_TILE_MODE0:
1089                 case mmGB_TILE_MODE1:
1090                 case mmGB_TILE_MODE2:
1091                 case mmGB_TILE_MODE3:
1092                 case mmGB_TILE_MODE4:
1093                 case mmGB_TILE_MODE5:
1094                 case mmGB_TILE_MODE6:
1095                 case mmGB_TILE_MODE7:
1096                 case mmGB_TILE_MODE8:
1097                 case mmGB_TILE_MODE9:
1098                 case mmGB_TILE_MODE10:
1099                 case mmGB_TILE_MODE11:
1100                 case mmGB_TILE_MODE12:
1101                 case mmGB_TILE_MODE13:
1102                 case mmGB_TILE_MODE14:
1103                 case mmGB_TILE_MODE15:
1104                 case mmGB_TILE_MODE16:
1105                 case mmGB_TILE_MODE17:
1106                 case mmGB_TILE_MODE18:
1107                 case mmGB_TILE_MODE19:
1108                 case mmGB_TILE_MODE20:
1109                 case mmGB_TILE_MODE21:
1110                 case mmGB_TILE_MODE22:
1111                 case mmGB_TILE_MODE23:
1112                 case mmGB_TILE_MODE24:
1113                 case mmGB_TILE_MODE25:
1114                 case mmGB_TILE_MODE26:
1115                 case mmGB_TILE_MODE27:
1116                 case mmGB_TILE_MODE28:
1117                 case mmGB_TILE_MODE29:
1118                 case mmGB_TILE_MODE30:
1119                 case mmGB_TILE_MODE31:
1120                         idx = (reg_offset - mmGB_TILE_MODE0);
1121                         return adev->gfx.config.tile_mode_array[idx];
1122                 default:
1123                         return RREG32(reg_offset);
1124                 }
1125         }
1126 }
1127 static int si_read_register(struct amdgpu_device *adev, u32 se_num,
1128                              u32 sh_num, u32 reg_offset, u32 *value)
1129 {
1130         uint32_t i;
1131
1132         *value = 0;
1133         for (i = 0; i < ARRAY_SIZE(si_allowed_read_registers); i++) {
1134                 bool indexed = si_allowed_read_registers[i].grbm_indexed;
1135
1136                 if (reg_offset != si_allowed_read_registers[i].reg_offset)
1137                         continue;
1138
1139                 *value = si_get_register_value(adev, indexed, se_num, sh_num,
1140                                                reg_offset);
1141                 return 0;
1142         }
1143         return -EINVAL;
1144 }
1145
1146 static bool si_read_disabled_bios(struct amdgpu_device *adev)
1147 {
1148         u32 bus_cntl;
1149         u32 d1vga_control = 0;
1150         u32 d2vga_control = 0;
1151         u32 vga_render_control = 0;
1152         u32 rom_cntl;
1153         bool r;
1154
1155         bus_cntl = RREG32(R600_BUS_CNTL);
1156         if (adev->mode_info.num_crtc) {
1157                 d1vga_control = RREG32(AVIVO_D1VGA_CONTROL);
1158                 d2vga_control = RREG32(AVIVO_D2VGA_CONTROL);
1159                 vga_render_control = RREG32(VGA_RENDER_CONTROL);
1160         }
1161         rom_cntl = RREG32(R600_ROM_CNTL);
1162
1163         /* enable the rom */
1164         WREG32(R600_BUS_CNTL, (bus_cntl & ~R600_BIOS_ROM_DIS));
1165         if (adev->mode_info.num_crtc) {
1166                 /* Disable VGA mode */
1167                 WREG32(AVIVO_D1VGA_CONTROL,
1168                        (d1vga_control & ~(AVIVO_DVGA_CONTROL_MODE_ENABLE |
1169                                           AVIVO_DVGA_CONTROL_TIMING_SELECT)));
1170                 WREG32(AVIVO_D2VGA_CONTROL,
1171                        (d2vga_control & ~(AVIVO_DVGA_CONTROL_MODE_ENABLE |
1172                                           AVIVO_DVGA_CONTROL_TIMING_SELECT)));
1173                 WREG32(VGA_RENDER_CONTROL,
1174                        (vga_render_control & C_000300_VGA_VSTATUS_CNTL));
1175         }
1176         WREG32(R600_ROM_CNTL, rom_cntl | R600_SCK_OVERWRITE);
1177
1178         r = amdgpu_read_bios(adev);
1179
1180         /* restore regs */
1181         WREG32(R600_BUS_CNTL, bus_cntl);
1182         if (adev->mode_info.num_crtc) {
1183                 WREG32(AVIVO_D1VGA_CONTROL, d1vga_control);
1184                 WREG32(AVIVO_D2VGA_CONTROL, d2vga_control);
1185                 WREG32(VGA_RENDER_CONTROL, vga_render_control);
1186         }
1187         WREG32(R600_ROM_CNTL, rom_cntl);
1188         return r;
1189 }
1190
1191 #define mmROM_INDEX 0x2A
1192 #define mmROM_DATA  0x2B
1193
1194 static bool si_read_bios_from_rom(struct amdgpu_device *adev,
1195                                   u8 *bios, u32 length_bytes)
1196 {
1197         u32 *dw_ptr;
1198         u32 i, length_dw;
1199
1200         if (bios == NULL)
1201                 return false;
1202         if (length_bytes == 0)
1203                 return false;
1204         /* APU vbios image is part of sbios image */
1205         if (adev->flags & AMD_IS_APU)
1206                 return false;
1207
1208         dw_ptr = (u32 *)bios;
1209         length_dw = ALIGN(length_bytes, 4) / 4;
1210         /* set rom index to 0 */
1211         WREG32(mmROM_INDEX, 0);
1212         for (i = 0; i < length_dw; i++)
1213                 dw_ptr[i] = RREG32(mmROM_DATA);
1214
1215         return true;
1216 }
1217
1218 //xxx: not implemented
1219 static int si_asic_reset(struct amdgpu_device *adev)
1220 {
1221         return 0;
1222 }
1223
1224 static bool si_asic_supports_baco(struct amdgpu_device *adev)
1225 {
1226         return false;
1227 }
1228
1229 static enum amd_reset_method
1230 si_asic_reset_method(struct amdgpu_device *adev)
1231 {
1232         return AMD_RESET_METHOD_LEGACY;
1233 }
1234
1235 static u32 si_get_config_memsize(struct amdgpu_device *adev)
1236 {
1237         return RREG32(mmCONFIG_MEMSIZE);
1238 }
1239
1240 static void si_vga_set_state(struct amdgpu_device *adev, bool state)
1241 {
1242         uint32_t temp;
1243
1244         temp = RREG32(CONFIG_CNTL);
1245         if (state == false) {
1246                 temp &= ~(1<<0);
1247                 temp |= (1<<1);
1248         } else {
1249                 temp &= ~(1<<1);
1250         }
1251         WREG32(CONFIG_CNTL, temp);
1252 }
1253
1254 static u32 si_get_xclk(struct amdgpu_device *adev)
1255 {
1256         u32 reference_clock = adev->clock.spll.reference_freq;
1257         u32 tmp;
1258
1259         tmp = RREG32(CG_CLKPIN_CNTL_2);
1260         if (tmp & MUX_TCLK_TO_XCLK)
1261                 return TCLK;
1262
1263         tmp = RREG32(CG_CLKPIN_CNTL);
1264         if (tmp & XTALIN_DIVIDE)
1265                 return reference_clock / 4;
1266
1267         return reference_clock;
1268 }
1269
1270 //xxx:not implemented
1271 static int si_set_uvd_clocks(struct amdgpu_device *adev, u32 vclk, u32 dclk)
1272 {
1273         return 0;
1274 }
1275
1276 static void si_flush_hdp(struct amdgpu_device *adev, struct amdgpu_ring *ring)
1277 {
1278         if (!ring || !ring->funcs->emit_wreg) {
1279                 WREG32(mmHDP_MEM_COHERENCY_FLUSH_CNTL, 1);
1280                 RREG32(mmHDP_MEM_COHERENCY_FLUSH_CNTL);
1281         } else {
1282                 amdgpu_ring_emit_wreg(ring, mmHDP_MEM_COHERENCY_FLUSH_CNTL, 1);
1283         }
1284 }
1285
1286 static void si_invalidate_hdp(struct amdgpu_device *adev,
1287                               struct amdgpu_ring *ring)
1288 {
1289         if (!ring || !ring->funcs->emit_wreg) {
1290                 WREG32(mmHDP_DEBUG0, 1);
1291                 RREG32(mmHDP_DEBUG0);
1292         } else {
1293                 amdgpu_ring_emit_wreg(ring, mmHDP_DEBUG0, 1);
1294         }
1295 }
1296
1297 static bool si_need_full_reset(struct amdgpu_device *adev)
1298 {
1299         /* change this when we support soft reset */
1300         return true;
1301 }
1302
1303 static bool si_need_reset_on_init(struct amdgpu_device *adev)
1304 {
1305         return false;
1306 }
1307
1308 static int si_get_pcie_lanes(struct amdgpu_device *adev)
1309 {
1310         u32 link_width_cntl;
1311
1312         if (adev->flags & AMD_IS_APU)
1313                 return 0;
1314
1315         link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
1316
1317         switch ((link_width_cntl & LC_LINK_WIDTH_RD_MASK) >> LC_LINK_WIDTH_RD_SHIFT) {
1318         case LC_LINK_WIDTH_X1:
1319                 return 1;
1320         case LC_LINK_WIDTH_X2:
1321                 return 2;
1322         case LC_LINK_WIDTH_X4:
1323                 return 4;
1324         case LC_LINK_WIDTH_X8:
1325                 return 8;
1326         case LC_LINK_WIDTH_X0:
1327         case LC_LINK_WIDTH_X16:
1328         default:
1329                 return 16;
1330         }
1331 }
1332
1333 static void si_set_pcie_lanes(struct amdgpu_device *adev, int lanes)
1334 {
1335         u32 link_width_cntl, mask;
1336
1337         if (adev->flags & AMD_IS_APU)
1338                 return;
1339
1340         switch (lanes) {
1341         case 0:
1342                 mask = LC_LINK_WIDTH_X0;
1343                 break;
1344         case 1:
1345                 mask = LC_LINK_WIDTH_X1;
1346                 break;
1347         case 2:
1348                 mask = LC_LINK_WIDTH_X2;
1349                 break;
1350         case 4:
1351                 mask = LC_LINK_WIDTH_X4;
1352                 break;
1353         case 8:
1354                 mask = LC_LINK_WIDTH_X8;
1355                 break;
1356         case 16:
1357                 mask = LC_LINK_WIDTH_X16;
1358                 break;
1359         default:
1360                 DRM_ERROR("invalid pcie lane request: %d\n", lanes);
1361                 return;
1362         }
1363
1364         link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
1365         link_width_cntl &= ~LC_LINK_WIDTH_MASK;
1366         link_width_cntl |= mask << LC_LINK_WIDTH_SHIFT;
1367         link_width_cntl |= (LC_RECONFIG_NOW |
1368                             LC_RECONFIG_ARC_MISSING_ESCAPE);
1369
1370         WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
1371 }
1372
1373 static void si_get_pcie_usage(struct amdgpu_device *adev, uint64_t *count0,
1374                               uint64_t *count1)
1375 {
1376         uint32_t perfctr = 0;
1377         uint64_t cnt0_of, cnt1_of;
1378         int tmp;
1379
1380         /* This reports 0 on APUs, so return to avoid writing/reading registers
1381          * that may or may not be different from their GPU counterparts
1382          */
1383         if (adev->flags & AMD_IS_APU)
1384                 return;
1385
1386         /* Set the 2 events that we wish to watch, defined above */
1387         /* Reg 40 is # received msgs, Reg 104 is # of posted requests sent */
1388         perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK, EVENT0_SEL, 40);
1389         perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK, EVENT1_SEL, 104);
1390
1391         /* Write to enable desired perf counters */
1392         WREG32_PCIE(ixPCIE_PERF_CNTL_TXCLK, perfctr);
1393         /* Zero out and enable the perf counters
1394          * Write 0x5:
1395          * Bit 0 = Start all counters(1)
1396          * Bit 2 = Global counter reset enable(1)
1397          */
1398         WREG32_PCIE(ixPCIE_PERF_COUNT_CNTL, 0x00000005);
1399
1400         msleep(1000);
1401
1402         /* Load the shadow and disable the perf counters
1403          * Write 0x2:
1404          * Bit 0 = Stop counters(0)
1405          * Bit 1 = Load the shadow counters(1)
1406          */
1407         WREG32_PCIE(ixPCIE_PERF_COUNT_CNTL, 0x00000002);
1408
1409         /* Read register values to get any >32bit overflow */
1410         tmp = RREG32_PCIE(ixPCIE_PERF_CNTL_TXCLK);
1411         cnt0_of = REG_GET_FIELD(tmp, PCIE_PERF_CNTL_TXCLK, COUNTER0_UPPER);
1412         cnt1_of = REG_GET_FIELD(tmp, PCIE_PERF_CNTL_TXCLK, COUNTER1_UPPER);
1413
1414         /* Get the values and add the overflow */
1415         *count0 = RREG32_PCIE(ixPCIE_PERF_COUNT0_TXCLK) | (cnt0_of << 32);
1416         *count1 = RREG32_PCIE(ixPCIE_PERF_COUNT1_TXCLK) | (cnt1_of << 32);
1417 }
1418
1419 static uint64_t si_get_pcie_replay_count(struct amdgpu_device *adev)
1420 {
1421         uint64_t nak_r, nak_g;
1422
1423         /* Get the number of NAKs received and generated */
1424         nak_r = RREG32_PCIE(ixPCIE_RX_NUM_NAK);
1425         nak_g = RREG32_PCIE(ixPCIE_RX_NUM_NAK_GENERATED);
1426
1427         /* Add the total number of NAKs, i.e the number of replays */
1428         return (nak_r + nak_g);
1429 }
1430
1431 static const struct amdgpu_asic_funcs si_asic_funcs =
1432 {
1433         .read_disabled_bios = &si_read_disabled_bios,
1434         .read_bios_from_rom = &si_read_bios_from_rom,
1435         .read_register = &si_read_register,
1436         .reset = &si_asic_reset,
1437         .reset_method = &si_asic_reset_method,
1438         .set_vga_state = &si_vga_set_state,
1439         .get_xclk = &si_get_xclk,
1440         .set_uvd_clocks = &si_set_uvd_clocks,
1441         .set_vce_clocks = NULL,
1442         .get_pcie_lanes = &si_get_pcie_lanes,
1443         .set_pcie_lanes = &si_set_pcie_lanes,
1444         .get_config_memsize = &si_get_config_memsize,
1445         .flush_hdp = &si_flush_hdp,
1446         .invalidate_hdp = &si_invalidate_hdp,
1447         .need_full_reset = &si_need_full_reset,
1448         .get_pcie_usage = &si_get_pcie_usage,
1449         .need_reset_on_init = &si_need_reset_on_init,
1450         .get_pcie_replay_count = &si_get_pcie_replay_count,
1451         .supports_baco = &si_asic_supports_baco,
1452 };
1453
1454 static uint32_t si_get_rev_id(struct amdgpu_device *adev)
1455 {
1456         return (RREG32(CC_DRM_ID_STRAPS) & CC_DRM_ID_STRAPS__ATI_REV_ID_MASK)
1457                 >> CC_DRM_ID_STRAPS__ATI_REV_ID__SHIFT;
1458 }
1459
1460 static int si_common_early_init(void *handle)
1461 {
1462         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1463
1464         adev->smc_rreg = &si_smc_rreg;
1465         adev->smc_wreg = &si_smc_wreg;
1466         adev->pcie_rreg = &si_pcie_rreg;
1467         adev->pcie_wreg = &si_pcie_wreg;
1468         adev->pciep_rreg = &si_pciep_rreg;
1469         adev->pciep_wreg = &si_pciep_wreg;
1470         adev->uvd_ctx_rreg = si_uvd_ctx_rreg;
1471         adev->uvd_ctx_wreg = si_uvd_ctx_wreg;
1472         adev->didt_rreg = NULL;
1473         adev->didt_wreg = NULL;
1474
1475         adev->asic_funcs = &si_asic_funcs;
1476
1477         adev->rev_id = si_get_rev_id(adev);
1478         adev->external_rev_id = 0xFF;
1479         switch (adev->asic_type) {
1480         case CHIP_TAHITI:
1481                 adev->cg_flags =
1482                         AMD_CG_SUPPORT_GFX_MGCG |
1483                         AMD_CG_SUPPORT_GFX_MGLS |
1484                         /*AMD_CG_SUPPORT_GFX_CGCG |*/
1485                         AMD_CG_SUPPORT_GFX_CGLS |
1486                         AMD_CG_SUPPORT_GFX_CGTS |
1487                         AMD_CG_SUPPORT_GFX_CP_LS |
1488                         AMD_CG_SUPPORT_MC_MGCG |
1489                         AMD_CG_SUPPORT_SDMA_MGCG |
1490                         AMD_CG_SUPPORT_BIF_LS |
1491                         AMD_CG_SUPPORT_VCE_MGCG |
1492                         AMD_CG_SUPPORT_UVD_MGCG |
1493                         AMD_CG_SUPPORT_HDP_LS |
1494                         AMD_CG_SUPPORT_HDP_MGCG;
1495                 adev->pg_flags = 0;
1496                 adev->external_rev_id = (adev->rev_id == 0) ? 1 :
1497                                         (adev->rev_id == 1) ? 5 : 6;
1498                 break;
1499         case CHIP_PITCAIRN:
1500                 adev->cg_flags =
1501                         AMD_CG_SUPPORT_GFX_MGCG |
1502                         AMD_CG_SUPPORT_GFX_MGLS |
1503                         /*AMD_CG_SUPPORT_GFX_CGCG |*/
1504                         AMD_CG_SUPPORT_GFX_CGLS |
1505                         AMD_CG_SUPPORT_GFX_CGTS |
1506                         AMD_CG_SUPPORT_GFX_CP_LS |
1507                         AMD_CG_SUPPORT_GFX_RLC_LS |
1508                         AMD_CG_SUPPORT_MC_LS |
1509                         AMD_CG_SUPPORT_MC_MGCG |
1510                         AMD_CG_SUPPORT_SDMA_MGCG |
1511                         AMD_CG_SUPPORT_BIF_LS |
1512                         AMD_CG_SUPPORT_VCE_MGCG |
1513                         AMD_CG_SUPPORT_UVD_MGCG |
1514                         AMD_CG_SUPPORT_HDP_LS |
1515                         AMD_CG_SUPPORT_HDP_MGCG;
1516                 adev->pg_flags = 0;
1517                 adev->external_rev_id = adev->rev_id + 20;
1518                 break;
1519
1520         case CHIP_VERDE:
1521                 adev->cg_flags =
1522                         AMD_CG_SUPPORT_GFX_MGCG |
1523                         AMD_CG_SUPPORT_GFX_MGLS |
1524                         AMD_CG_SUPPORT_GFX_CGLS |
1525                         AMD_CG_SUPPORT_GFX_CGTS |
1526                         AMD_CG_SUPPORT_GFX_CGTS_LS |
1527                         AMD_CG_SUPPORT_GFX_CP_LS |
1528                         AMD_CG_SUPPORT_MC_LS |
1529                         AMD_CG_SUPPORT_MC_MGCG |
1530                         AMD_CG_SUPPORT_SDMA_MGCG |
1531                         AMD_CG_SUPPORT_SDMA_LS |
1532                         AMD_CG_SUPPORT_BIF_LS |
1533                         AMD_CG_SUPPORT_VCE_MGCG |
1534                         AMD_CG_SUPPORT_UVD_MGCG |
1535                         AMD_CG_SUPPORT_HDP_LS |
1536                         AMD_CG_SUPPORT_HDP_MGCG;
1537                 adev->pg_flags = 0;
1538                 //???
1539                 adev->external_rev_id = adev->rev_id + 40;
1540                 break;
1541         case CHIP_OLAND:
1542                 adev->cg_flags =
1543                         AMD_CG_SUPPORT_GFX_MGCG |
1544                         AMD_CG_SUPPORT_GFX_MGLS |
1545                         /*AMD_CG_SUPPORT_GFX_CGCG |*/
1546                         AMD_CG_SUPPORT_GFX_CGLS |
1547                         AMD_CG_SUPPORT_GFX_CGTS |
1548                         AMD_CG_SUPPORT_GFX_CP_LS |
1549                         AMD_CG_SUPPORT_GFX_RLC_LS |
1550                         AMD_CG_SUPPORT_MC_LS |
1551                         AMD_CG_SUPPORT_MC_MGCG |
1552                         AMD_CG_SUPPORT_SDMA_MGCG |
1553                         AMD_CG_SUPPORT_BIF_LS |
1554                         AMD_CG_SUPPORT_UVD_MGCG |
1555                         AMD_CG_SUPPORT_HDP_LS |
1556                         AMD_CG_SUPPORT_HDP_MGCG;
1557                 adev->pg_flags = 0;
1558                 adev->external_rev_id = 60;
1559                 break;
1560         case CHIP_HAINAN:
1561                 adev->cg_flags =
1562                         AMD_CG_SUPPORT_GFX_MGCG |
1563                         AMD_CG_SUPPORT_GFX_MGLS |
1564                         /*AMD_CG_SUPPORT_GFX_CGCG |*/
1565                         AMD_CG_SUPPORT_GFX_CGLS |
1566                         AMD_CG_SUPPORT_GFX_CGTS |
1567                         AMD_CG_SUPPORT_GFX_CP_LS |
1568                         AMD_CG_SUPPORT_GFX_RLC_LS |
1569                         AMD_CG_SUPPORT_MC_LS |
1570                         AMD_CG_SUPPORT_MC_MGCG |
1571                         AMD_CG_SUPPORT_SDMA_MGCG |
1572                         AMD_CG_SUPPORT_BIF_LS |
1573                         AMD_CG_SUPPORT_HDP_LS |
1574                         AMD_CG_SUPPORT_HDP_MGCG;
1575                 adev->pg_flags = 0;
1576                 adev->external_rev_id = 70;
1577                 break;
1578
1579         default:
1580                 return -EINVAL;
1581         }
1582
1583         return 0;
1584 }
1585
1586 static int si_common_sw_init(void *handle)
1587 {
1588         return 0;
1589 }
1590
1591 static int si_common_sw_fini(void *handle)
1592 {
1593         return 0;
1594 }
1595
1596
1597 static void si_init_golden_registers(struct amdgpu_device *adev)
1598 {
1599         switch (adev->asic_type) {
1600         case CHIP_TAHITI:
1601                 amdgpu_device_program_register_sequence(adev,
1602                                                         tahiti_golden_registers,
1603                                                         ARRAY_SIZE(tahiti_golden_registers));
1604                 amdgpu_device_program_register_sequence(adev,
1605                                                         tahiti_golden_rlc_registers,
1606                                                         ARRAY_SIZE(tahiti_golden_rlc_registers));
1607                 amdgpu_device_program_register_sequence(adev,
1608                                                         tahiti_mgcg_cgcg_init,
1609                                                         ARRAY_SIZE(tahiti_mgcg_cgcg_init));
1610                 amdgpu_device_program_register_sequence(adev,
1611                                                         tahiti_golden_registers2,
1612                                                         ARRAY_SIZE(tahiti_golden_registers2));
1613                 break;
1614         case CHIP_PITCAIRN:
1615                 amdgpu_device_program_register_sequence(adev,
1616                                                         pitcairn_golden_registers,
1617                                                         ARRAY_SIZE(pitcairn_golden_registers));
1618                 amdgpu_device_program_register_sequence(adev,
1619                                                         pitcairn_golden_rlc_registers,
1620                                                         ARRAY_SIZE(pitcairn_golden_rlc_registers));
1621                 amdgpu_device_program_register_sequence(adev,
1622                                                         pitcairn_mgcg_cgcg_init,
1623                                                         ARRAY_SIZE(pitcairn_mgcg_cgcg_init));
1624                 break;
1625         case CHIP_VERDE:
1626                 amdgpu_device_program_register_sequence(adev,
1627                                                         verde_golden_registers,
1628                                                         ARRAY_SIZE(verde_golden_registers));
1629                 amdgpu_device_program_register_sequence(adev,
1630                                                         verde_golden_rlc_registers,
1631                                                         ARRAY_SIZE(verde_golden_rlc_registers));
1632                 amdgpu_device_program_register_sequence(adev,
1633                                                         verde_mgcg_cgcg_init,
1634                                                         ARRAY_SIZE(verde_mgcg_cgcg_init));
1635                 amdgpu_device_program_register_sequence(adev,
1636                                                         verde_pg_init,
1637                                                         ARRAY_SIZE(verde_pg_init));
1638                 break;
1639         case CHIP_OLAND:
1640                 amdgpu_device_program_register_sequence(adev,
1641                                                         oland_golden_registers,
1642                                                         ARRAY_SIZE(oland_golden_registers));
1643                 amdgpu_device_program_register_sequence(adev,
1644                                                         oland_golden_rlc_registers,
1645                                                         ARRAY_SIZE(oland_golden_rlc_registers));
1646                 amdgpu_device_program_register_sequence(adev,
1647                                                         oland_mgcg_cgcg_init,
1648                                                         ARRAY_SIZE(oland_mgcg_cgcg_init));
1649                 break;
1650         case CHIP_HAINAN:
1651                 amdgpu_device_program_register_sequence(adev,
1652                                                         hainan_golden_registers,
1653                                                         ARRAY_SIZE(hainan_golden_registers));
1654                 amdgpu_device_program_register_sequence(adev,
1655                                                         hainan_golden_registers2,
1656                                                         ARRAY_SIZE(hainan_golden_registers2));
1657                 amdgpu_device_program_register_sequence(adev,
1658                                                         hainan_mgcg_cgcg_init,
1659                                                         ARRAY_SIZE(hainan_mgcg_cgcg_init));
1660                 break;
1661
1662
1663         default:
1664                 BUG();
1665         }
1666 }
1667
1668 static void si_pcie_gen3_enable(struct amdgpu_device *adev)
1669 {
1670         struct pci_dev *root = adev->pdev->bus->self;
1671         u32 speed_cntl, current_data_rate;
1672         int i;
1673         u16 tmp16;
1674
1675         if (pci_is_root_bus(adev->pdev->bus))
1676                 return;
1677
1678         if (amdgpu_pcie_gen2 == 0)
1679                 return;
1680
1681         if (adev->flags & AMD_IS_APU)
1682                 return;
1683
1684         if (!(adev->pm.pcie_gen_mask & (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
1685                                         CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)))
1686                 return;
1687
1688         speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
1689         current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
1690                 LC_CURRENT_DATA_RATE_SHIFT;
1691         if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
1692                 if (current_data_rate == 2) {
1693                         DRM_INFO("PCIE gen 3 link speeds already enabled\n");
1694                         return;
1695                 }
1696                 DRM_INFO("enabling PCIE gen 3 link speeds, disable with amdgpu.pcie_gen2=0\n");
1697         } else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2) {
1698                 if (current_data_rate == 1) {
1699                         DRM_INFO("PCIE gen 2 link speeds already enabled\n");
1700                         return;
1701                 }
1702                 DRM_INFO("enabling PCIE gen 2 link speeds, disable with amdgpu.pcie_gen2=0\n");
1703         }
1704
1705         if (!pci_is_pcie(root) || !pci_is_pcie(adev->pdev))
1706                 return;
1707
1708         if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
1709                 if (current_data_rate != 2) {
1710                         u16 bridge_cfg, gpu_cfg;
1711                         u16 bridge_cfg2, gpu_cfg2;
1712                         u32 max_lw, current_lw, tmp;
1713
1714                         pcie_capability_read_word(root, PCI_EXP_LNKCTL,
1715                                                   &bridge_cfg);
1716                         pcie_capability_read_word(adev->pdev, PCI_EXP_LNKCTL,
1717                                                   &gpu_cfg);
1718
1719                         tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
1720                         pcie_capability_write_word(root, PCI_EXP_LNKCTL, tmp16);
1721
1722                         tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
1723                         pcie_capability_write_word(adev->pdev, PCI_EXP_LNKCTL,
1724                                                    tmp16);
1725
1726                         tmp = RREG32_PCIE(PCIE_LC_STATUS1);
1727                         max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
1728                         current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
1729
1730                         if (current_lw < max_lw) {
1731                                 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
1732                                 if (tmp & LC_RENEGOTIATION_SUPPORT) {
1733                                         tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
1734                                         tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
1735                                         tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
1736                                         WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
1737                                 }
1738                         }
1739
1740                         for (i = 0; i < 10; i++) {
1741                                 pcie_capability_read_word(adev->pdev,
1742                                                           PCI_EXP_DEVSTA,
1743                                                           &tmp16);
1744                                 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
1745                                         break;
1746
1747                                 pcie_capability_read_word(root, PCI_EXP_LNKCTL,
1748                                                           &bridge_cfg);
1749                                 pcie_capability_read_word(adev->pdev,
1750                                                           PCI_EXP_LNKCTL,
1751                                                           &gpu_cfg);
1752
1753                                 pcie_capability_read_word(root, PCI_EXP_LNKCTL2,
1754                                                           &bridge_cfg2);
1755                                 pcie_capability_read_word(adev->pdev,
1756                                                           PCI_EXP_LNKCTL2,
1757                                                           &gpu_cfg2);
1758
1759                                 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
1760                                 tmp |= LC_SET_QUIESCE;
1761                                 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
1762
1763                                 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
1764                                 tmp |= LC_REDO_EQ;
1765                                 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
1766
1767                                 mdelay(100);
1768
1769                                 pcie_capability_read_word(root, PCI_EXP_LNKCTL,
1770                                                           &tmp16);
1771                                 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
1772                                 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
1773                                 pcie_capability_write_word(root, PCI_EXP_LNKCTL,
1774                                                            tmp16);
1775
1776                                 pcie_capability_read_word(adev->pdev,
1777                                                           PCI_EXP_LNKCTL,
1778                                                           &tmp16);
1779                                 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
1780                                 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
1781                                 pcie_capability_write_word(adev->pdev,
1782                                                            PCI_EXP_LNKCTL,
1783                                                            tmp16);
1784
1785                                 pcie_capability_read_word(root, PCI_EXP_LNKCTL2,
1786                                                           &tmp16);
1787                                 tmp16 &= ~(PCI_EXP_LNKCTL2_ENTER_COMP |
1788                                            PCI_EXP_LNKCTL2_TX_MARGIN);
1789                                 tmp16 |= (bridge_cfg2 &
1790                                           (PCI_EXP_LNKCTL2_ENTER_COMP |
1791                                            PCI_EXP_LNKCTL2_TX_MARGIN));
1792                                 pcie_capability_write_word(root,
1793                                                            PCI_EXP_LNKCTL2,
1794                                                            tmp16);
1795
1796                                 pcie_capability_read_word(adev->pdev,
1797                                                           PCI_EXP_LNKCTL2,
1798                                                           &tmp16);
1799                                 tmp16 &= ~(PCI_EXP_LNKCTL2_ENTER_COMP |
1800                                            PCI_EXP_LNKCTL2_TX_MARGIN);
1801                                 tmp16 |= (gpu_cfg2 &
1802                                           (PCI_EXP_LNKCTL2_ENTER_COMP |
1803                                            PCI_EXP_LNKCTL2_TX_MARGIN));
1804                                 pcie_capability_write_word(adev->pdev,
1805                                                            PCI_EXP_LNKCTL2,
1806                                                            tmp16);
1807
1808                                 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
1809                                 tmp &= ~LC_SET_QUIESCE;
1810                                 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
1811                         }
1812                 }
1813         }
1814
1815         speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
1816         speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
1817         WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
1818
1819         pcie_capability_read_word(adev->pdev, PCI_EXP_LNKCTL2, &tmp16);
1820         tmp16 &= ~PCI_EXP_LNKCTL2_TLS;
1821
1822         if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)
1823                 tmp16 |= PCI_EXP_LNKCTL2_TLS_8_0GT; /* gen3 */
1824         else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2)
1825                 tmp16 |= PCI_EXP_LNKCTL2_TLS_5_0GT; /* gen2 */
1826         else
1827                 tmp16 |= PCI_EXP_LNKCTL2_TLS_2_5GT; /* gen1 */
1828         pcie_capability_write_word(adev->pdev, PCI_EXP_LNKCTL2, tmp16);
1829
1830         speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
1831         speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
1832         WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
1833
1834         for (i = 0; i < adev->usec_timeout; i++) {
1835                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
1836                 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
1837                         break;
1838                 udelay(1);
1839         }
1840 }
1841
1842 static inline u32 si_pif_phy0_rreg(struct amdgpu_device *adev, u32 reg)
1843 {
1844         unsigned long flags;
1845         u32 r;
1846
1847         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
1848         WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
1849         r = RREG32(EVERGREEN_PIF_PHY0_DATA);
1850         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
1851         return r;
1852 }
1853
1854 static inline void si_pif_phy0_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
1855 {
1856         unsigned long flags;
1857
1858         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
1859         WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
1860         WREG32(EVERGREEN_PIF_PHY0_DATA, (v));
1861         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
1862 }
1863
1864 static inline u32 si_pif_phy1_rreg(struct amdgpu_device *adev, u32 reg)
1865 {
1866         unsigned long flags;
1867         u32 r;
1868
1869         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
1870         WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
1871         r = RREG32(EVERGREEN_PIF_PHY1_DATA);
1872         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
1873         return r;
1874 }
1875
1876 static inline void si_pif_phy1_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
1877 {
1878         unsigned long flags;
1879
1880         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
1881         WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
1882         WREG32(EVERGREEN_PIF_PHY1_DATA, (v));
1883         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
1884 }
1885 static void si_program_aspm(struct amdgpu_device *adev)
1886 {
1887         u32 data, orig;
1888         bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
1889         bool disable_clkreq = false;
1890
1891         if (amdgpu_aspm == 0)
1892                 return;
1893
1894         if (adev->flags & AMD_IS_APU)
1895                 return;
1896         orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
1897         data &= ~LC_XMIT_N_FTS_MASK;
1898         data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
1899         if (orig != data)
1900                 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
1901
1902         orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
1903         data |= LC_GO_TO_RECOVERY;
1904         if (orig != data)
1905                 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
1906
1907         orig = data = RREG32_PCIE(PCIE_P_CNTL);
1908         data |= P_IGNORE_EDB_ERR;
1909         if (orig != data)
1910                 WREG32_PCIE(PCIE_P_CNTL, data);
1911
1912         orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
1913         data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
1914         data |= LC_PMI_TO_L1_DIS;
1915         if (!disable_l0s)
1916                 data |= LC_L0S_INACTIVITY(7);
1917
1918         if (!disable_l1) {
1919                 data |= LC_L1_INACTIVITY(7);
1920                 data &= ~LC_PMI_TO_L1_DIS;
1921                 if (orig != data)
1922                         WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
1923
1924                 if (!disable_plloff_in_l1) {
1925                         bool clk_req_support;
1926
1927                         orig = data = si_pif_phy0_rreg(adev,PB0_PIF_PWRDOWN_0);
1928                         data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
1929                         data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
1930                         if (orig != data)
1931                                 si_pif_phy0_wreg(adev,PB0_PIF_PWRDOWN_0, data);
1932
1933                         orig = data = si_pif_phy0_rreg(adev,PB0_PIF_PWRDOWN_1);
1934                         data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
1935                         data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
1936                         if (orig != data)
1937                                 si_pif_phy0_wreg(adev,PB0_PIF_PWRDOWN_1, data);
1938
1939                         orig = data = si_pif_phy1_rreg(adev,PB1_PIF_PWRDOWN_0);
1940                         data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
1941                         data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
1942                         if (orig != data)
1943                                 si_pif_phy1_wreg(adev,PB1_PIF_PWRDOWN_0, data);
1944
1945                         orig = data = si_pif_phy1_rreg(adev,PB1_PIF_PWRDOWN_1);
1946                         data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
1947                         data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
1948                         if (orig != data)
1949                                 si_pif_phy1_wreg(adev,PB1_PIF_PWRDOWN_1, data);
1950
1951                         if ((adev->asic_type != CHIP_OLAND) && (adev->asic_type != CHIP_HAINAN)) {
1952                                 orig = data = si_pif_phy0_rreg(adev,PB0_PIF_PWRDOWN_0);
1953                                 data &= ~PLL_RAMP_UP_TIME_0_MASK;
1954                                 if (orig != data)
1955                                         si_pif_phy0_wreg(adev,PB0_PIF_PWRDOWN_0, data);
1956
1957                                 orig = data = si_pif_phy0_rreg(adev,PB0_PIF_PWRDOWN_1);
1958                                 data &= ~PLL_RAMP_UP_TIME_1_MASK;
1959                                 if (orig != data)
1960                                         si_pif_phy0_wreg(adev,PB0_PIF_PWRDOWN_1, data);
1961
1962                                 orig = data = si_pif_phy0_rreg(adev,PB0_PIF_PWRDOWN_2);
1963                                 data &= ~PLL_RAMP_UP_TIME_2_MASK;
1964                                 if (orig != data)
1965                                         si_pif_phy0_wreg(adev,PB0_PIF_PWRDOWN_2, data);
1966
1967                                 orig = data = si_pif_phy0_rreg(adev,PB0_PIF_PWRDOWN_3);
1968                                 data &= ~PLL_RAMP_UP_TIME_3_MASK;
1969                                 if (orig != data)
1970                                         si_pif_phy0_wreg(adev,PB0_PIF_PWRDOWN_3, data);
1971
1972                                 orig = data = si_pif_phy1_rreg(adev,PB1_PIF_PWRDOWN_0);
1973                                 data &= ~PLL_RAMP_UP_TIME_0_MASK;
1974                                 if (orig != data)
1975                                         si_pif_phy1_wreg(adev,PB1_PIF_PWRDOWN_0, data);
1976
1977                                 orig = data = si_pif_phy1_rreg(adev,PB1_PIF_PWRDOWN_1);
1978                                 data &= ~PLL_RAMP_UP_TIME_1_MASK;
1979                                 if (orig != data)
1980                                         si_pif_phy1_wreg(adev,PB1_PIF_PWRDOWN_1, data);
1981
1982                                 orig = data = si_pif_phy1_rreg(adev,PB1_PIF_PWRDOWN_2);
1983                                 data &= ~PLL_RAMP_UP_TIME_2_MASK;
1984                                 if (orig != data)
1985                                         si_pif_phy1_wreg(adev,PB1_PIF_PWRDOWN_2, data);
1986
1987                                 orig = data = si_pif_phy1_rreg(adev,PB1_PIF_PWRDOWN_3);
1988                                 data &= ~PLL_RAMP_UP_TIME_3_MASK;
1989                                 if (orig != data)
1990                                         si_pif_phy1_wreg(adev,PB1_PIF_PWRDOWN_3, data);
1991                         }
1992                         orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
1993                         data &= ~LC_DYN_LANES_PWR_STATE_MASK;
1994                         data |= LC_DYN_LANES_PWR_STATE(3);
1995                         if (orig != data)
1996                                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
1997
1998                         orig = data = si_pif_phy0_rreg(adev,PB0_PIF_CNTL);
1999                         data &= ~LS2_EXIT_TIME_MASK;
2000                         if ((adev->asic_type == CHIP_OLAND) || (adev->asic_type == CHIP_HAINAN))
2001                                 data |= LS2_EXIT_TIME(5);
2002                         if (orig != data)
2003                                 si_pif_phy0_wreg(adev,PB0_PIF_CNTL, data);
2004
2005                         orig = data = si_pif_phy1_rreg(adev,PB1_PIF_CNTL);
2006                         data &= ~LS2_EXIT_TIME_MASK;
2007                         if ((adev->asic_type == CHIP_OLAND) || (adev->asic_type == CHIP_HAINAN))
2008                                 data |= LS2_EXIT_TIME(5);
2009                         if (orig != data)
2010                                 si_pif_phy1_wreg(adev,PB1_PIF_CNTL, data);
2011
2012                         if (!disable_clkreq &&
2013                             !pci_is_root_bus(adev->pdev->bus)) {
2014                                 struct pci_dev *root = adev->pdev->bus->self;
2015                                 u32 lnkcap;
2016
2017                                 clk_req_support = false;
2018                                 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
2019                                 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
2020                                         clk_req_support = true;
2021                         } else {
2022                                 clk_req_support = false;
2023                         }
2024
2025                         if (clk_req_support) {
2026                                 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
2027                                 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
2028                                 if (orig != data)
2029                                         WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
2030
2031                                 orig = data = RREG32(THM_CLK_CNTL);
2032                                 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
2033                                 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
2034                                 if (orig != data)
2035                                         WREG32(THM_CLK_CNTL, data);
2036
2037                                 orig = data = RREG32(MISC_CLK_CNTL);
2038                                 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
2039                                 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
2040                                 if (orig != data)
2041                                         WREG32(MISC_CLK_CNTL, data);
2042
2043                                 orig = data = RREG32(CG_CLKPIN_CNTL);
2044                                 data &= ~BCLK_AS_XCLK;
2045                                 if (orig != data)
2046                                         WREG32(CG_CLKPIN_CNTL, data);
2047
2048                                 orig = data = RREG32(CG_CLKPIN_CNTL_2);
2049                                 data &= ~FORCE_BIF_REFCLK_EN;
2050                                 if (orig != data)
2051                                         WREG32(CG_CLKPIN_CNTL_2, data);
2052
2053                                 orig = data = RREG32(MPLL_BYPASSCLK_SEL);
2054                                 data &= ~MPLL_CLKOUT_SEL_MASK;
2055                                 data |= MPLL_CLKOUT_SEL(4);
2056                                 if (orig != data)
2057                                         WREG32(MPLL_BYPASSCLK_SEL, data);
2058
2059                                 orig = data = RREG32(SPLL_CNTL_MODE);
2060                                 data &= ~SPLL_REFCLK_SEL_MASK;
2061                                 if (orig != data)
2062                                         WREG32(SPLL_CNTL_MODE, data);
2063                         }
2064                 }
2065         } else {
2066                 if (orig != data)
2067                         WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
2068         }
2069
2070         orig = data = RREG32_PCIE(PCIE_CNTL2);
2071         data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
2072         if (orig != data)
2073                 WREG32_PCIE(PCIE_CNTL2, data);
2074
2075         if (!disable_l0s) {
2076                 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
2077                 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
2078                         data = RREG32_PCIE(PCIE_LC_STATUS1);
2079                         if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
2080                                 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
2081                                 data &= ~LC_L0S_INACTIVITY_MASK;
2082                                 if (orig != data)
2083                                         WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
2084                         }
2085                 }
2086         }
2087 }
2088
2089 static void si_fix_pci_max_read_req_size(struct amdgpu_device *adev)
2090 {
2091         int readrq;
2092         u16 v;
2093
2094         readrq = pcie_get_readrq(adev->pdev);
2095         v = ffs(readrq) - 8;
2096         if ((v == 0) || (v == 6) || (v == 7))
2097                 pcie_set_readrq(adev->pdev, 512);
2098 }
2099
2100 static int si_common_hw_init(void *handle)
2101 {
2102         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2103
2104         si_fix_pci_max_read_req_size(adev);
2105         si_init_golden_registers(adev);
2106         si_pcie_gen3_enable(adev);
2107         si_program_aspm(adev);
2108
2109         return 0;
2110 }
2111
2112 static int si_common_hw_fini(void *handle)
2113 {
2114         return 0;
2115 }
2116
2117 static int si_common_suspend(void *handle)
2118 {
2119         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2120
2121         return si_common_hw_fini(adev);
2122 }
2123
2124 static int si_common_resume(void *handle)
2125 {
2126         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2127
2128         return si_common_hw_init(adev);
2129 }
2130
2131 static bool si_common_is_idle(void *handle)
2132 {
2133         return true;
2134 }
2135
2136 static int si_common_wait_for_idle(void *handle)
2137 {
2138         return 0;
2139 }
2140
2141 static int si_common_soft_reset(void *handle)
2142 {
2143         return 0;
2144 }
2145
2146 static int si_common_set_clockgating_state(void *handle,
2147                                             enum amd_clockgating_state state)
2148 {
2149         return 0;
2150 }
2151
2152 static int si_common_set_powergating_state(void *handle,
2153                                             enum amd_powergating_state state)
2154 {
2155         return 0;
2156 }
2157
2158 static const struct amd_ip_funcs si_common_ip_funcs = {
2159         .name = "si_common",
2160         .early_init = si_common_early_init,
2161         .late_init = NULL,
2162         .sw_init = si_common_sw_init,
2163         .sw_fini = si_common_sw_fini,
2164         .hw_init = si_common_hw_init,
2165         .hw_fini = si_common_hw_fini,
2166         .suspend = si_common_suspend,
2167         .resume = si_common_resume,
2168         .is_idle = si_common_is_idle,
2169         .wait_for_idle = si_common_wait_for_idle,
2170         .soft_reset = si_common_soft_reset,
2171         .set_clockgating_state = si_common_set_clockgating_state,
2172         .set_powergating_state = si_common_set_powergating_state,
2173 };
2174
2175 static const struct amdgpu_ip_block_version si_common_ip_block =
2176 {
2177         .type = AMD_IP_BLOCK_TYPE_COMMON,
2178         .major = 1,
2179         .minor = 0,
2180         .rev = 0,
2181         .funcs = &si_common_ip_funcs,
2182 };
2183
2184 int si_set_ip_blocks(struct amdgpu_device *adev)
2185 {
2186         switch (adev->asic_type) {
2187         case CHIP_VERDE:
2188         case CHIP_TAHITI:
2189         case CHIP_PITCAIRN:
2190                 amdgpu_device_ip_block_add(adev, &si_common_ip_block);
2191                 amdgpu_device_ip_block_add(adev, &gmc_v6_0_ip_block);
2192                 amdgpu_device_ip_block_add(adev, &si_ih_ip_block);
2193                 amdgpu_device_ip_block_add(adev, &gfx_v6_0_ip_block);
2194                 amdgpu_device_ip_block_add(adev, &si_dma_ip_block);
2195                 amdgpu_device_ip_block_add(adev, &si_smu_ip_block);
2196                 if (adev->enable_virtual_display)
2197                         amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block);
2198                 else
2199                         amdgpu_device_ip_block_add(adev, &dce_v6_0_ip_block);
2200                 amdgpu_device_ip_block_add(adev, &uvd_v3_1_ip_block);
2201                 /* amdgpu_device_ip_block_add(adev, &vce_v1_0_ip_block); */
2202                 break;
2203         case CHIP_OLAND:
2204                 amdgpu_device_ip_block_add(adev, &si_common_ip_block);
2205                 amdgpu_device_ip_block_add(adev, &gmc_v6_0_ip_block);
2206                 amdgpu_device_ip_block_add(adev, &si_ih_ip_block);
2207                 amdgpu_device_ip_block_add(adev, &gfx_v6_0_ip_block);
2208                 amdgpu_device_ip_block_add(adev, &si_dma_ip_block);
2209                 amdgpu_device_ip_block_add(adev, &si_smu_ip_block);
2210                 if (adev->enable_virtual_display)
2211                         amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block);
2212                 else
2213                         amdgpu_device_ip_block_add(adev, &dce_v6_4_ip_block);
2214                 amdgpu_device_ip_block_add(adev, &uvd_v3_1_ip_block);
2215                 /* amdgpu_device_ip_block_add(adev, &vce_v1_0_ip_block); */
2216                 break;
2217         case CHIP_HAINAN:
2218                 amdgpu_device_ip_block_add(adev, &si_common_ip_block);
2219                 amdgpu_device_ip_block_add(adev, &gmc_v6_0_ip_block);
2220                 amdgpu_device_ip_block_add(adev, &si_ih_ip_block);
2221                 amdgpu_device_ip_block_add(adev, &gfx_v6_0_ip_block);
2222                 amdgpu_device_ip_block_add(adev, &si_dma_ip_block);
2223                 amdgpu_device_ip_block_add(adev, &si_smu_ip_block);
2224                 if (adev->enable_virtual_display)
2225                         amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block);
2226                 break;
2227         default:
2228                 BUG();
2229         }
2230         return 0;
2231 }
2232