drivers/gpu/drm/amd/amdgpu/si.c

   1 /*
   2  * Copyright 2015 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23
  24 #include <linux/firmware.h>
  25 #include <linux/slab.h>
  26 #include <linux/module.h>
  27 #include <linux/pci.h>
  28
  29 #include "amdgpu.h"
  30 #include "amdgpu_atombios.h"
  31 #include "amdgpu_ih.h"
  32 #include "amdgpu_uvd.h"
  33 #include "amdgpu_vce.h"
  34 #include "atom.h"
  35 #include "amd_pcie.h"
  36 #include "si_dpm.h"
  37 #include "sid.h"
  38 #include "si_ih.h"
  39 #include "gfx_v6_0.h"
  40 #include "gmc_v6_0.h"
  41 #include "si_dma.h"
  42 #include "dce_v6_0.h"
  43 #include "si.h"
  44 #include "uvd_v3_1.h"
  45 #include "dce_virtual.h"
  46 #include "gca/gfx_6_0_d.h"
  47 #include "oss/oss_1_0_d.h"
  48 #include "oss/oss_1_0_sh_mask.h"
  49 #include "gmc/gmc_6_0_d.h"
  50 #include "dce/dce_6_0_d.h"
  51 #include "uvd/uvd_4_0_d.h"
  52 #include "bif/bif_3_0_d.h"
  53 #include "bif/bif_3_0_sh_mask.h"
  54
  55 static const u32 tahiti_golden_registers[] =
  56 {
  57         mmAZALIA_SCLK_CONTROL, 0x00000030, 0x00000011,
  58         mmCB_HW_CONTROL, 0x00010000, 0x00018208,
  59         mmDB_DEBUG, 0xffffffff, 0x00000000,
  60         mmDB_DEBUG2, 0xf00fffff, 0x00000400,
  61         mmDB_DEBUG3, 0x0002021c, 0x00020200,
  62         mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
  63         0x340c, 0x000000c0, 0x00800040,
  64         0x360c, 0x000000c0, 0x00800040,
  65         mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
  66         mmFBC_MISC, 0x00200000, 0x50100000,
  67         mmDIG0_HDMI_CONTROL, 0x31000311, 0x00000011,
  68         mmMC_ARB_WTM_CNTL_RD, 0x00000003, 0x000007ff,
  69         mmMC_XPB_P2P_BAR_CFG, 0x000007ff, 0x00000000,
  70         mmPA_CL_ENHANCE, 0xf000001f, 0x00000007,
  71         mmPA_SC_FORCE_EOV_MAX_CNTS, 0xffffffff, 0x00ffffff,
  72         mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
  73         mmPA_SC_MODE_CNTL_1, 0x07ffffff, 0x4e000000,
  74         mmPA_SC_RASTER_CONFIG, 0x3f3f3fff, 0x2a00126a,
  75         0x000c, 0xffffffff, 0x0040,
  76         0x000d, 0x00000040, 0x00004040,
  77         mmSPI_CONFIG_CNTL, 0x07ffffff, 0x03000000,
  78         mmSQ_DED_CNT, 0x01ff1f3f, 0x00000000,
  79         mmSQ_SEC_CNT, 0x01ff1f3f, 0x00000000,
  80         mmSX_DEBUG_1, 0x0000007f, 0x00000020,
  81         mmTA_CNTL_AUX, 0x00010000, 0x00010000,
  82         mmTCP_ADDR_CONFIG, 0x00000200, 0x000002fb,
  83         mmTCP_CHAN_STEER_HI, 0xffffffff, 0x0000543b,
  84         mmTCP_CHAN_STEER_LO, 0xffffffff, 0xa9210876,
  85         mmVGT_FIFO_DEPTHS, 0xffffffff, 0x000fff40,
  86         mmVGT_GS_VERTEX_REUSE, 0x0000001f, 0x00000010,
  87         mmVM_CONTEXT0_CNTL, 0x20000000, 0x20fffed8,
  88         mmVM_L2_CG, 0x000c0fc0, 0x000c0400,
  89         mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0xffffffff,
  90         mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
  91         mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
  92         mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
  93 };
  94
  95 static const u32 tahiti_golden_registers2[] =
  96 {
  97         mmMCIF_MEM_CONTROL, 0x00000001, 0x00000001,
  98 };
  99
 100 static const u32 tahiti_golden_rlc_registers[] =
 101 {
 102         mmGB_ADDR_CONFIG, 0xffffffff, 0x12011003,
 103         mmRLC_LB_PARAMS, 0xffffffff, 0x00601005,
 104         0x311f, 0xffffffff, 0x10104040,
 105         0x3122, 0xffffffff, 0x0100000a,
 106         mmRLC_LB_CNTR_MAX, 0xffffffff, 0x00000800,
 107         mmRLC_LB_CNTL, 0xffffffff, 0x800000f4,
 108         mmUVD_CGC_GATE, 0x00000008, 0x00000000,
 109 };
 110
 111 static const u32 pitcairn_golden_registers[] =
 112 {
 113         mmAZALIA_SCLK_CONTROL, 0x00000030, 0x00000011,
 114         mmCB_HW_CONTROL, 0x00010000, 0x00018208,
 115         mmDB_DEBUG, 0xffffffff, 0x00000000,
 116         mmDB_DEBUG2, 0xf00fffff, 0x00000400,
 117         mmDB_DEBUG3, 0x0002021c, 0x00020200,
 118         mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
 119         0x340c, 0x000300c0, 0x00800040,
 120         0x360c, 0x000300c0, 0x00800040,
 121         mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
 122         mmFBC_MISC, 0x00200000, 0x50100000,
 123         mmDIG0_HDMI_CONTROL, 0x31000311, 0x00000011,
 124         mmMC_SEQ_PMG_PG_HWCNTL, 0x00073ffe, 0x000022a2,
 125         mmMC_XPB_P2P_BAR_CFG, 0x000007ff, 0x00000000,
 126         mmPA_CL_ENHANCE, 0xf000001f, 0x00000007,
 127         mmPA_SC_FORCE_EOV_MAX_CNTS, 0xffffffff, 0x00ffffff,
 128         mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
 129         mmPA_SC_MODE_CNTL_1, 0x07ffffff, 0x4e000000,
 130         mmPA_SC_RASTER_CONFIG, 0x3f3f3fff, 0x2a00126a,
 131         0x000c, 0xffffffff, 0x0040,
 132         0x000d, 0x00000040, 0x00004040,
 133         mmSPI_CONFIG_CNTL, 0x07ffffff, 0x03000000,
 134         mmSX_DEBUG_1, 0x0000007f, 0x00000020,
 135         mmTA_CNTL_AUX, 0x00010000, 0x00010000,
 136         mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f7,
 137         mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,
 138         mmTCP_CHAN_STEER_LO, 0xffffffff, 0x32761054,
 139         mmVGT_GS_VERTEX_REUSE, 0x0000001f, 0x00000010,
 140         mmVM_L2_CG, 0x000c0fc0, 0x000c0400,
 141         mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0xffffffff,
 142         mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 143         mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 144         mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 145 };
 146
 147 static const u32 pitcairn_golden_rlc_registers[] =
 148 {
 149         mmGB_ADDR_CONFIG, 0xffffffff, 0x12011003,
 150         mmRLC_LB_PARAMS, 0xffffffff, 0x00601004,
 151         0x311f, 0xffffffff, 0x10102020,
 152         0x3122, 0xffffffff, 0x01000020,
 153         mmRLC_LB_CNTR_MAX, 0xffffffff, 0x00000800,
 154         mmRLC_LB_CNTL, 0xffffffff, 0x800000a4,
 155 };
 156
 157 static const u32 verde_pg_init[] =
 158 {
 159         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x40000,
 160         mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x200010ff,
 161         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 162         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 163         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 164         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 165         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 166         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x7007,
 167         mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x300010ff,
 168         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 169         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 170         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 171         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 172         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 173         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x400000,
 174         mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x100010ff,
 175         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 176         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 177         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 178         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 179         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 180         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x120200,
 181         mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x500010ff,
 182         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 183         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 184         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 185         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 186         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 187         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x1e1e16,
 188         mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x600010ff,
 189         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 190         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 191         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 192         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 193         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 194         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x171f1e,
 195         mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x700010ff,
 196         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 197         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 198         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 199         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 200         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 201         mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 202         mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x9ff,
 203         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x0,
 204         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x10000800,
 205         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xf,
 206         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xf,
 207         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x4,
 208         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1000051e,
 209         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xffff,
 210         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xffff,
 211         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x8,
 212         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x80500,
 213         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x12,
 214         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x9050c,
 215         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x1d,
 216         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xb052c,
 217         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x2a,
 218         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1053e,
 219         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x2d,
 220         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x10546,
 221         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x30,
 222         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xa054e,
 223         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x3c,
 224         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1055f,
 225         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x3f,
 226         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x10567,
 227         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x42,
 228         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1056f,
 229         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x45,
 230         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x10572,
 231         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x48,
 232         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x20575,
 233         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x4c,
 234         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x190801,
 235         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x67,
 236         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1082a,
 237         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x6a,
 238         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1b082d,
 239         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x87,
 240         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x310851,
 241         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xba,
 242         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x891,
 243         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xbc,
 244         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x893,
 245         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xbe,
 246         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x20895,
 247         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xc2,
 248         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x20899,
 249         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xc6,
 250         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x2089d,
 251         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xca,
 252         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x8a1,
 253         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xcc,
 254         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x8a3,
 255         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xce,
 256         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x308a5,
 257         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xd3,
 258         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x6d08cd,
 259         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x142,
 260         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x2000095a,
 261         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1,
 262         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x144,
 263         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x301f095b,
 264         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x165,
 265         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xc094d,
 266         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x173,
 267         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xf096d,
 268         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x184,
 269         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x15097f,
 270         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x19b,
 271         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xc0998,
 272         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x1a9,
 273         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x409a7,
 274         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x1af,
 275         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xcdc,
 276         mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x1b1,
 277         mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x800,
 278         mmGMCON_RENG_EXECUTE, 0xffffffff, 0x6c9b2000,
 279         mmGMCON_MISC2, 0xfc00, 0x2000,
 280         mmGMCON_MISC3, 0xffffffff, 0xfc0,
 281         mmMC_PMG_AUTO_CFG, 0x00000100, 0x100,
 282 };
 283
 284 static const u32 verde_golden_rlc_registers[] =
 285 {
 286         mmGB_ADDR_CONFIG, 0xffffffff, 0x02010002,
 287         mmRLC_LB_PARAMS, 0xffffffff, 0x033f1005,
 288         0x311f, 0xffffffff, 0x10808020,
 289         0x3122, 0xffffffff, 0x00800008,
 290         mmRLC_LB_CNTR_MAX, 0xffffffff, 0x00001000,
 291         mmRLC_LB_CNTL, 0xffffffff, 0x80010014,
 292 };
 293
 294 static const u32 verde_golden_registers[] =
 295 {
 296         mmAZALIA_SCLK_CONTROL, 0x00000030, 0x00000011,
 297         mmCB_HW_CONTROL, 0x00010000, 0x00018208,
 298         mmDB_DEBUG, 0xffffffff, 0x00000000,
 299         mmDB_DEBUG2, 0xf00fffff, 0x00000400,
 300         mmDB_DEBUG3, 0x0002021c, 0x00020200,
 301         mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
 302         0x340c, 0x000300c0, 0x00800040,
 303         0x360c, 0x000300c0, 0x00800040,
 304         mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
 305         mmFBC_MISC, 0x00200000, 0x50100000,
 306         mmDIG0_HDMI_CONTROL, 0x31000311, 0x00000011,
 307         mmMC_SEQ_PMG_PG_HWCNTL, 0x00073ffe, 0x000022a2,
 308         mmMC_XPB_P2P_BAR_CFG, 0x000007ff, 0x00000000,
 309         mmPA_CL_ENHANCE, 0xf000001f, 0x00000007,
 310         mmPA_SC_FORCE_EOV_MAX_CNTS, 0xffffffff, 0x00ffffff,
 311         mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
 312         mmPA_SC_MODE_CNTL_1, 0x07ffffff, 0x4e000000,
 313         mmPA_SC_RASTER_CONFIG, 0x3f3f3fff, 0x0000124a,
 314         0x000c, 0xffffffff, 0x0040,
 315         0x000d, 0x00000040, 0x00004040,
 316         mmSPI_CONFIG_CNTL, 0x07ffffff, 0x03000000,
 317         mmSQ_DED_CNT, 0x01ff1f3f, 0x00000000,
 318         mmSQ_SEC_CNT, 0x01ff1f3f, 0x00000000,
 319         mmSX_DEBUG_1, 0x0000007f, 0x00000020,
 320         mmTA_CNTL_AUX, 0x00010000, 0x00010000,
 321         mmTCP_ADDR_CONFIG, 0x000003ff, 0x00000003,
 322         mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,
 323         mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00001032,
 324         mmVGT_GS_VERTEX_REUSE, 0x0000001f, 0x00000010,
 325         mmVM_L2_CG, 0x000c0fc0, 0x000c0400,
 326         mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0xffffffff,
 327         mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 328         mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 329         mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 330 };
 331
 332 static const u32 oland_golden_registers[] =
 333 {
 334         mmAZALIA_SCLK_CONTROL, 0x00000030, 0x00000011,
 335         mmCB_HW_CONTROL, 0x00010000, 0x00018208,
 336         mmDB_DEBUG, 0xffffffff, 0x00000000,
 337         mmDB_DEBUG2, 0xf00fffff, 0x00000400,
 338         mmDB_DEBUG3, 0x0002021c, 0x00020200,
 339         mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
 340         0x340c, 0x000300c0, 0x00800040,
 341         0x360c, 0x000300c0, 0x00800040,
 342         mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
 343         mmFBC_MISC, 0x00200000, 0x50100000,
 344         mmDIG0_HDMI_CONTROL, 0x31000311, 0x00000011,
 345         mmMC_SEQ_PMG_PG_HWCNTL, 0x00073ffe, 0x000022a2,
 346         mmMC_XPB_P2P_BAR_CFG, 0x000007ff, 0x00000000,
 347         mmPA_CL_ENHANCE, 0xf000001f, 0x00000007,
 348         mmPA_SC_FORCE_EOV_MAX_CNTS, 0xffffffff, 0x00ffffff,
 349         mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
 350         mmPA_SC_MODE_CNTL_1, 0x07ffffff, 0x4e000000,
 351         mmPA_SC_RASTER_CONFIG, 0x3f3f3fff, 0x00000082,
 352         0x000c, 0xffffffff, 0x0040,
 353         0x000d, 0x00000040, 0x00004040,
 354         mmSPI_CONFIG_CNTL, 0x07ffffff, 0x03000000,
 355         mmSX_DEBUG_1, 0x0000007f, 0x00000020,
 356         mmTA_CNTL_AUX, 0x00010000, 0x00010000,
 357         mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f3,
 358         mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,
 359         mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003210,
 360         mmVGT_GS_VERTEX_REUSE, 0x0000001f, 0x00000010,
 361         mmVM_L2_CG, 0x000c0fc0, 0x000c0400,
 362         mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0xffffffff,
 363         mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 364         mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 365         mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 366
 367 };
 368
 369 static const u32 oland_golden_rlc_registers[] =
 370 {
 371         mmGB_ADDR_CONFIG, 0xffffffff, 0x02010002,
 372         mmRLC_LB_PARAMS, 0xffffffff, 0x00601005,
 373         0x311f, 0xffffffff, 0x10104040,
 374         0x3122, 0xffffffff, 0x0100000a,
 375         mmRLC_LB_CNTR_MAX, 0xffffffff, 0x00000800,
 376         mmRLC_LB_CNTL, 0xffffffff, 0x800000f4,
 377 };
 378
 379 static const u32 hainan_golden_registers[] =
 380 {
 381         0x17bc, 0x00000030, 0x00000011,
 382         mmCB_HW_CONTROL, 0x00010000, 0x00018208,
 383         mmDB_DEBUG, 0xffffffff, 0x00000000,
 384         mmDB_DEBUG2, 0xf00fffff, 0x00000400,
 385         mmDB_DEBUG3, 0x0002021c, 0x00020200,
 386         0x031e, 0x00000080, 0x00000000,
 387         0x3430, 0xff000fff, 0x00000100,
 388         0x340c, 0x000300c0, 0x00800040,
 389         0x3630, 0xff000fff, 0x00000100,
 390         0x360c, 0x000300c0, 0x00800040,
 391         0x16ec, 0x000000f0, 0x00000070,
 392         0x16f0, 0x00200000, 0x50100000,
 393         0x1c0c, 0x31000311, 0x00000011,
 394         mmMC_SEQ_PMG_PG_HWCNTL, 0x00073ffe, 0x000022a2,
 395         mmMC_XPB_P2P_BAR_CFG, 0x000007ff, 0x00000000,
 396         mmPA_CL_ENHANCE, 0xf000001f, 0x00000007,
 397         mmPA_SC_FORCE_EOV_MAX_CNTS, 0xffffffff, 0x00ffffff,
 398         mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
 399         mmPA_SC_MODE_CNTL_1, 0x07ffffff, 0x4e000000,
 400         mmPA_SC_RASTER_CONFIG, 0x3f3f3fff, 0x00000000,
 401         0x000c, 0xffffffff, 0x0040,
 402         0x000d, 0x00000040, 0x00004040,
 403         mmSPI_CONFIG_CNTL, 0x03e00000, 0x03600000,
 404         mmSX_DEBUG_1, 0x0000007f, 0x00000020,
 405         mmTA_CNTL_AUX, 0x00010000, 0x00010000,
 406         mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f1,
 407         mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,
 408         mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003210,
 409         mmVGT_GS_VERTEX_REUSE, 0x0000001f, 0x00000010,
 410         mmVM_L2_CG, 0x000c0fc0, 0x000c0400,
 411         mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0xffffffff,
 412         mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 413         mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 414         mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 415 };
 416
 417 static const u32 hainan_golden_registers2[] =
 418 {
 419         mmGB_ADDR_CONFIG, 0xffffffff, 0x2011003,
 420 };
 421
 422 static const u32 tahiti_mgcg_cgcg_init[] =
 423 {
 424         mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xfffffffc,
 425         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 426         mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 427         mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
 428         mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
 429         mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
 430         mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
 431         mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
 432         mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
 433         mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
 434         mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
 435         mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
 436         mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
 437         mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
 438         mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
 439         mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
 440         mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
 441         mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
 442         mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
 443         mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
 444         mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
 445         mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
 446         mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
 447         mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 448         mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 449         mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
 450         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 451         0x2458, 0xffffffff, 0x00010000,
 452         0x2459, 0xffffffff, 0x00030002,
 453         0x245a, 0xffffffff, 0x00040007,
 454         0x245b, 0xffffffff, 0x00060005,
 455         0x245c, 0xffffffff, 0x00090008,
 456         0x245d, 0xffffffff, 0x00020001,
 457         0x245e, 0xffffffff, 0x00040003,
 458         0x245f, 0xffffffff, 0x00000007,
 459         0x2460, 0xffffffff, 0x00060005,
 460         0x2461, 0xffffffff, 0x00090008,
 461         0x2462, 0xffffffff, 0x00030002,
 462         0x2463, 0xffffffff, 0x00050004,
 463         0x2464, 0xffffffff, 0x00000008,
 464         0x2465, 0xffffffff, 0x00070006,
 465         0x2466, 0xffffffff, 0x000a0009,
 466         0x2467, 0xffffffff, 0x00040003,
 467         0x2468, 0xffffffff, 0x00060005,
 468         0x2469, 0xffffffff, 0x00000009,
 469         0x246a, 0xffffffff, 0x00080007,
 470         0x246b, 0xffffffff, 0x000b000a,
 471         0x246c, 0xffffffff, 0x00050004,
 472         0x246d, 0xffffffff, 0x00070006,
 473         0x246e, 0xffffffff, 0x0008000b,
 474         0x246f, 0xffffffff, 0x000a0009,
 475         0x2470, 0xffffffff, 0x000d000c,
 476         0x2471, 0xffffffff, 0x00060005,
 477         0x2472, 0xffffffff, 0x00080007,
 478         0x2473, 0xffffffff, 0x0000000b,
 479         0x2474, 0xffffffff, 0x000a0009,
 480         0x2475, 0xffffffff, 0x000d000c,
 481         0x2476, 0xffffffff, 0x00070006,
 482         0x2477, 0xffffffff, 0x00090008,
 483         0x2478, 0xffffffff, 0x0000000c,
 484         0x2479, 0xffffffff, 0x000b000a,
 485         0x247a, 0xffffffff, 0x000e000d,
 486         0x247b, 0xffffffff, 0x00080007,
 487         0x247c, 0xffffffff, 0x000a0009,
 488         0x247d, 0xffffffff, 0x0000000d,
 489         0x247e, 0xffffffff, 0x000c000b,
 490         0x247f, 0xffffffff, 0x000f000e,
 491         0x2480, 0xffffffff, 0x00090008,
 492         0x2481, 0xffffffff, 0x000b000a,
 493         0x2482, 0xffffffff, 0x000c000f,
 494         0x2483, 0xffffffff, 0x000e000d,
 495         0x2484, 0xffffffff, 0x00110010,
 496         0x2485, 0xffffffff, 0x000a0009,
 497         0x2486, 0xffffffff, 0x000c000b,
 498         0x2487, 0xffffffff, 0x0000000f,
 499         0x2488, 0xffffffff, 0x000e000d,
 500         0x2489, 0xffffffff, 0x00110010,
 501         0x248a, 0xffffffff, 0x000b000a,
 502         0x248b, 0xffffffff, 0x000d000c,
 503         0x248c, 0xffffffff, 0x00000010,
 504         0x248d, 0xffffffff, 0x000f000e,
 505         0x248e, 0xffffffff, 0x00120011,
 506         0x248f, 0xffffffff, 0x000c000b,
 507         0x2490, 0xffffffff, 0x000e000d,
 508         0x2491, 0xffffffff, 0x00000011,
 509         0x2492, 0xffffffff, 0x0010000f,
 510         0x2493, 0xffffffff, 0x00130012,
 511         0x2494, 0xffffffff, 0x000d000c,
 512         0x2495, 0xffffffff, 0x000f000e,
 513         0x2496, 0xffffffff, 0x00100013,
 514         0x2497, 0xffffffff, 0x00120011,
 515         0x2498, 0xffffffff, 0x00150014,
 516         0x2499, 0xffffffff, 0x000e000d,
 517         0x249a, 0xffffffff, 0x0010000f,
 518         0x249b, 0xffffffff, 0x00000013,
 519         0x249c, 0xffffffff, 0x00120011,
 520         0x249d, 0xffffffff, 0x00150014,
 521         0x249e, 0xffffffff, 0x000f000e,
 522         0x249f, 0xffffffff, 0x00110010,
 523         0x24a0, 0xffffffff, 0x00000014,
 524         0x24a1, 0xffffffff, 0x00130012,
 525         0x24a2, 0xffffffff, 0x00160015,
 526         0x24a3, 0xffffffff, 0x0010000f,
 527         0x24a4, 0xffffffff, 0x00120011,
 528         0x24a5, 0xffffffff, 0x00000015,
 529         0x24a6, 0xffffffff, 0x00140013,
 530         0x24a7, 0xffffffff, 0x00170016,
 531         mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,
 532         mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
 533         mmRLC_GCPM_GENERAL_3, 0xffffffff, 0x00000080,
 534         mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,
 535         0x000c, 0xffffffff, 0x0000001c,
 536         0x000d, 0x000f0000, 0x000f0000,
 537         0x0583, 0xffffffff, 0x00000100,
 538         mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,
 539         mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,
 540         mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
 541         mmMC_CITF_MISC_WR_CG, 0x000c0000, 0x000c0000,
 542         mmMC_CITF_MISC_RD_CG, 0x000c0000, 0x000c0000,
 543         mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
 544         0x157a, 0x00000001, 0x00000001,
 545         mmHDP_MEM_POWER_LS, 0x00000001, 0x00000001,
 546         mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
 547         mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
 548         0x3430, 0xfffffff0, 0x00000100,
 549         0x3630, 0xfffffff0, 0x00000100,
 550 };
 551 static const u32 pitcairn_mgcg_cgcg_init[] =
 552 {
 553         mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xfffffffc,
 554         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 555         mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 556         mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
 557         mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
 558         mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
 559         mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
 560         mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
 561         mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
 562         mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
 563         mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
 564         mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
 565         mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
 566         mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
 567         mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
 568         mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
 569         mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
 570         mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
 571         mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
 572         mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
 573         mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
 574         mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
 575         mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
 576         mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 577         mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 578         mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
 579         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 580         0x2458, 0xffffffff, 0x00010000,
 581         0x2459, 0xffffffff, 0x00030002,
 582         0x245a, 0xffffffff, 0x00040007,
 583         0x245b, 0xffffffff, 0x00060005,
 584         0x245c, 0xffffffff, 0x00090008,
 585         0x245d, 0xffffffff, 0x00020001,
 586         0x245e, 0xffffffff, 0x00040003,
 587         0x245f, 0xffffffff, 0x00000007,
 588         0x2460, 0xffffffff, 0x00060005,
 589         0x2461, 0xffffffff, 0x00090008,
 590         0x2462, 0xffffffff, 0x00030002,
 591         0x2463, 0xffffffff, 0x00050004,
 592         0x2464, 0xffffffff, 0x00000008,
 593         0x2465, 0xffffffff, 0x00070006,
 594         0x2466, 0xffffffff, 0x000a0009,
 595         0x2467, 0xffffffff, 0x00040003,
 596         0x2468, 0xffffffff, 0x00060005,
 597         0x2469, 0xffffffff, 0x00000009,
 598         0x246a, 0xffffffff, 0x00080007,
 599         0x246b, 0xffffffff, 0x000b000a,
 600         0x246c, 0xffffffff, 0x00050004,
 601         0x246d, 0xffffffff, 0x00070006,
 602         0x246e, 0xffffffff, 0x0008000b,
 603         0x246f, 0xffffffff, 0x000a0009,
 604         0x2470, 0xffffffff, 0x000d000c,
 605         0x2480, 0xffffffff, 0x00090008,
 606         0x2481, 0xffffffff, 0x000b000a,
 607         0x2482, 0xffffffff, 0x000c000f,
 608         0x2483, 0xffffffff, 0x000e000d,
 609         0x2484, 0xffffffff, 0x00110010,
 610         0x2485, 0xffffffff, 0x000a0009,
 611         0x2486, 0xffffffff, 0x000c000b,
 612         0x2487, 0xffffffff, 0x0000000f,
 613         0x2488, 0xffffffff, 0x000e000d,
 614         0x2489, 0xffffffff, 0x00110010,
 615         0x248a, 0xffffffff, 0x000b000a,
 616         0x248b, 0xffffffff, 0x000d000c,
 617         0x248c, 0xffffffff, 0x00000010,
 618         0x248d, 0xffffffff, 0x000f000e,
 619         0x248e, 0xffffffff, 0x00120011,
 620         0x248f, 0xffffffff, 0x000c000b,
 621         0x2490, 0xffffffff, 0x000e000d,
 622         0x2491, 0xffffffff, 0x00000011,
 623         0x2492, 0xffffffff, 0x0010000f,
 624         0x2493, 0xffffffff, 0x00130012,
 625         0x2494, 0xffffffff, 0x000d000c,
 626         0x2495, 0xffffffff, 0x000f000e,
 627         0x2496, 0xffffffff, 0x00100013,
 628         0x2497, 0xffffffff, 0x00120011,
 629         0x2498, 0xffffffff, 0x00150014,
 630         mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,
 631         mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
 632         mmRLC_GCPM_GENERAL_3, 0xffffffff, 0x00000080,
 633         mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,
 634         0x000c, 0xffffffff, 0x0000001c,
 635         0x000d, 0x000f0000, 0x000f0000,
 636         0x0583, 0xffffffff, 0x00000100,
 637         mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,
 638         mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,
 639         mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
 640         mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
 641         0x157a, 0x00000001, 0x00000001,
 642         mmHDP_MEM_POWER_LS, 0x00000001, 0x00000001,
 643         mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
 644         mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
 645         0x3430, 0xfffffff0, 0x00000100,
 646         0x3630, 0xfffffff0, 0x00000100,
 647 };
 648
 649 static const u32 verde_mgcg_cgcg_init[] =
 650 {
 651         mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xfffffffc,
 652         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 653         mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 654         mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
 655         mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
 656         mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
 657         mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
 658         mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
 659         mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
 660         mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
 661         mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
 662         mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
 663         mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
 664         mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
 665         mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
 666         mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
 667         mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
 668         mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
 669         mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
 670         mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
 671         mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
 672         mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
 673         mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
 674         mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 675         mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 676         mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
 677         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 678         0x2458, 0xffffffff, 0x00010000,
 679         0x2459, 0xffffffff, 0x00030002,
 680         0x245a, 0xffffffff, 0x00040007,
 681         0x245b, 0xffffffff, 0x00060005,
 682         0x245c, 0xffffffff, 0x00090008,
 683         0x245d, 0xffffffff, 0x00020001,
 684         0x245e, 0xffffffff, 0x00040003,
 685         0x245f, 0xffffffff, 0x00000007,
 686         0x2460, 0xffffffff, 0x00060005,
 687         0x2461, 0xffffffff, 0x00090008,
 688         0x2462, 0xffffffff, 0x00030002,
 689         0x2463, 0xffffffff, 0x00050004,
 690         0x2464, 0xffffffff, 0x00000008,
 691         0x2465, 0xffffffff, 0x00070006,
 692         0x2466, 0xffffffff, 0x000a0009,
 693         0x2467, 0xffffffff, 0x00040003,
 694         0x2468, 0xffffffff, 0x00060005,
 695         0x2469, 0xffffffff, 0x00000009,
 696         0x246a, 0xffffffff, 0x00080007,
 697         0x246b, 0xffffffff, 0x000b000a,
 698         0x246c, 0xffffffff, 0x00050004,
 699         0x246d, 0xffffffff, 0x00070006,
 700         0x246e, 0xffffffff, 0x0008000b,
 701         0x246f, 0xffffffff, 0x000a0009,
 702         0x2470, 0xffffffff, 0x000d000c,
 703         0x2480, 0xffffffff, 0x00090008,
 704         0x2481, 0xffffffff, 0x000b000a,
 705         0x2482, 0xffffffff, 0x000c000f,
 706         0x2483, 0xffffffff, 0x000e000d,
 707         0x2484, 0xffffffff, 0x00110010,
 708         0x2485, 0xffffffff, 0x000a0009,
 709         0x2486, 0xffffffff, 0x000c000b,
 710         0x2487, 0xffffffff, 0x0000000f,
 711         0x2488, 0xffffffff, 0x000e000d,
 712         0x2489, 0xffffffff, 0x00110010,
 713         0x248a, 0xffffffff, 0x000b000a,
 714         0x248b, 0xffffffff, 0x000d000c,
 715         0x248c, 0xffffffff, 0x00000010,
 716         0x248d, 0xffffffff, 0x000f000e,
 717         0x248e, 0xffffffff, 0x00120011,
 718         0x248f, 0xffffffff, 0x000c000b,
 719         0x2490, 0xffffffff, 0x000e000d,
 720         0x2491, 0xffffffff, 0x00000011,
 721         0x2492, 0xffffffff, 0x0010000f,
 722         0x2493, 0xffffffff, 0x00130012,
 723         0x2494, 0xffffffff, 0x000d000c,
 724         0x2495, 0xffffffff, 0x000f000e,
 725         0x2496, 0xffffffff, 0x00100013,
 726         0x2497, 0xffffffff, 0x00120011,
 727         0x2498, 0xffffffff, 0x00150014,
 728         mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,
 729         mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
 730         mmRLC_GCPM_GENERAL_3, 0xffffffff, 0x00000080,
 731         mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,
 732         0x000c, 0xffffffff, 0x0000001c,
 733         0x000d, 0x000f0000, 0x000f0000,
 734         0x0583, 0xffffffff, 0x00000100,
 735         mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,
 736         mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,
 737         mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
 738         mmMC_CITF_MISC_WR_CG, 0x000c0000, 0x000c0000,
 739         mmMC_CITF_MISC_RD_CG, 0x000c0000, 0x000c0000,
 740         mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
 741         0x157a, 0x00000001, 0x00000001,
 742         mmHDP_MEM_POWER_LS, 0x00000001, 0x00000001,
 743         mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
 744         mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
 745         0x3430, 0xfffffff0, 0x00000100,
 746         0x3630, 0xfffffff0, 0x00000100,
 747 };
 748
 749 static const u32 oland_mgcg_cgcg_init[] =
 750 {
 751         mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xfffffffc,
 752         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 753         mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 754         mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
 755         mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
 756         mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
 757         mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
 758         mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
 759         mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
 760         mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
 761         mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
 762         mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
 763         mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
 764         mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
 765         mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
 766         mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
 767         mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
 768         mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
 769         mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
 770         mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
 771         mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
 772         mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
 773         mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
 774         mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 775         mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 776         mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
 777         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 778         0x2458, 0xffffffff, 0x00010000,
 779         0x2459, 0xffffffff, 0x00030002,
 780         0x245a, 0xffffffff, 0x00040007,
 781         0x245b, 0xffffffff, 0x00060005,
 782         0x245c, 0xffffffff, 0x00090008,
 783         0x245d, 0xffffffff, 0x00020001,
 784         0x245e, 0xffffffff, 0x00040003,
 785         0x245f, 0xffffffff, 0x00000007,
 786         0x2460, 0xffffffff, 0x00060005,
 787         0x2461, 0xffffffff, 0x00090008,
 788         0x2462, 0xffffffff, 0x00030002,
 789         0x2463, 0xffffffff, 0x00050004,
 790         0x2464, 0xffffffff, 0x00000008,
 791         0x2465, 0xffffffff, 0x00070006,
 792         0x2466, 0xffffffff, 0x000a0009,
 793         0x2467, 0xffffffff, 0x00040003,
 794         0x2468, 0xffffffff, 0x00060005,
 795         0x2469, 0xffffffff, 0x00000009,
 796         0x246a, 0xffffffff, 0x00080007,
 797         0x246b, 0xffffffff, 0x000b000a,
 798         0x246c, 0xffffffff, 0x00050004,
 799         0x246d, 0xffffffff, 0x00070006,
 800         0x246e, 0xffffffff, 0x0008000b,
 801         0x246f, 0xffffffff, 0x000a0009,
 802         0x2470, 0xffffffff, 0x000d000c,
 803         0x2471, 0xffffffff, 0x00060005,
 804         0x2472, 0xffffffff, 0x00080007,
 805         0x2473, 0xffffffff, 0x0000000b,
 806         0x2474, 0xffffffff, 0x000a0009,
 807         0x2475, 0xffffffff, 0x000d000c,
 808         mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,
 809         mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
 810         mmRLC_GCPM_GENERAL_3, 0xffffffff, 0x00000080,
 811         mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,
 812         0x000c, 0xffffffff, 0x0000001c,
 813         0x000d, 0x000f0000, 0x000f0000,
 814         0x0583, 0xffffffff, 0x00000100,
 815         mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,
 816         mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,
 817         mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
 818         mmMC_CITF_MISC_WR_CG, 0x000c0000, 0x000c0000,
 819         mmMC_CITF_MISC_RD_CG, 0x000c0000, 0x000c0000,
 820         mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
 821         0x157a, 0x00000001, 0x00000001,
 822         mmHDP_MEM_POWER_LS, 0x00000001, 0x00000001,
 823         mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
 824         mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
 825         0x3430, 0xfffffff0, 0x00000100,
 826         0x3630, 0xfffffff0, 0x00000100,
 827 };
 828
 829 static const u32 hainan_mgcg_cgcg_init[] =
 830 {
 831         mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xfffffffc,
 832         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 833         mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 834         mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
 835         mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
 836         mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
 837         mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
 838         mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
 839         mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
 840         mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
 841         mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
 842         mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
 843         mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
 844         mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
 845         mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
 846         mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
 847         mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
 848         mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
 849         mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
 850         mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
 851         mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
 852         mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
 853         mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
 854         mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 855         mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 856         mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
 857         mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 858         0x2458, 0xffffffff, 0x00010000,
 859         0x2459, 0xffffffff, 0x00030002,
 860         0x245a, 0xffffffff, 0x00040007,
 861         0x245b, 0xffffffff, 0x00060005,
 862         0x245c, 0xffffffff, 0x00090008,
 863         0x245d, 0xffffffff, 0x00020001,
 864         0x245e, 0xffffffff, 0x00040003,
 865         0x245f, 0xffffffff, 0x00000007,
 866         0x2460, 0xffffffff, 0x00060005,
 867         0x2461, 0xffffffff, 0x00090008,
 868         0x2462, 0xffffffff, 0x00030002,
 869         0x2463, 0xffffffff, 0x00050004,
 870         0x2464, 0xffffffff, 0x00000008,
 871         0x2465, 0xffffffff, 0x00070006,
 872         0x2466, 0xffffffff, 0x000a0009,
 873         0x2467, 0xffffffff, 0x00040003,
 874         0x2468, 0xffffffff, 0x00060005,
 875         0x2469, 0xffffffff, 0x00000009,
 876         0x246a, 0xffffffff, 0x00080007,
 877         0x246b, 0xffffffff, 0x000b000a,
 878         0x246c, 0xffffffff, 0x00050004,
 879         0x246d, 0xffffffff, 0x00070006,
 880         0x246e, 0xffffffff, 0x0008000b,
 881         0x246f, 0xffffffff, 0x000a0009,
 882         0x2470, 0xffffffff, 0x000d000c,
 883         0x2471, 0xffffffff, 0x00060005,
 884         0x2472, 0xffffffff, 0x00080007,
 885         0x2473, 0xffffffff, 0x0000000b,
 886         0x2474, 0xffffffff, 0x000a0009,
 887         0x2475, 0xffffffff, 0x000d000c,
 888         mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,
 889         mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
 890         mmRLC_GCPM_GENERAL_3, 0xffffffff, 0x00000080,
 891         mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,
 892         0x000c, 0xffffffff, 0x0000001c,
 893         0x000d, 0x000f0000, 0x000f0000,
 894         0x0583, 0xffffffff, 0x00000100,
 895         0x0409, 0xffffffff, 0x00000100,
 896         mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
 897         mmMC_CITF_MISC_WR_CG, 0x000c0000, 0x000c0000,
 898         mmMC_CITF_MISC_RD_CG, 0x000c0000, 0x000c0000,
 899         mmHDP_MEM_POWER_LS, 0x00000001, 0x00000001,
 900         mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
 901         mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
 902         0x3430, 0xfffffff0, 0x00000100,
 903         0x3630, 0xfffffff0, 0x00000100,
 904 };
 905
 906 static u32 si_pcie_rreg(struct amdgpu_device *adev, u32 reg)
 907 {
 908         unsigned long flags;
 909         u32 r;
 910
 911         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 912         WREG32(AMDGPU_PCIE_INDEX, reg);
 913         (void)RREG32(AMDGPU_PCIE_INDEX);
 914         r = RREG32(AMDGPU_PCIE_DATA);
 915         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 916         return r;
 917 }
 918
 919 static void si_pcie_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 920 {
 921         unsigned long flags;
 922
 923         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 924         WREG32(AMDGPU_PCIE_INDEX, reg);
 925         (void)RREG32(AMDGPU_PCIE_INDEX);
 926         WREG32(AMDGPU_PCIE_DATA, v);
 927         (void)RREG32(AMDGPU_PCIE_DATA);
 928         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 929 }
 930
 931 static u32 si_pciep_rreg(struct amdgpu_device *adev, u32 reg)
 932 {
 933         unsigned long flags;
 934         u32 r;
 935
 936         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 937         WREG32(PCIE_PORT_INDEX, ((reg) & 0xff));
 938         (void)RREG32(PCIE_PORT_INDEX);
 939         r = RREG32(PCIE_PORT_DATA);
 940         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 941         return r;
 942 }
 943
 944 static void si_pciep_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 945 {
 946         unsigned long flags;
 947
 948         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 949         WREG32(PCIE_PORT_INDEX, ((reg) & 0xff));
 950         (void)RREG32(PCIE_PORT_INDEX);
 951         WREG32(PCIE_PORT_DATA, (v));
 952         (void)RREG32(PCIE_PORT_DATA);
 953         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 954 }
 955
 956 static u32 si_smc_rreg(struct amdgpu_device *adev, u32 reg)
 957 {
 958         unsigned long flags;
 959         u32 r;
 960
 961         spin_lock_irqsave(&adev->smc_idx_lock, flags);
 962         WREG32(SMC_IND_INDEX_0, (reg));
 963         r = RREG32(SMC_IND_DATA_0);
 964         spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
 965         return r;
 966 }
 967
 968 static void si_smc_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 969 {
 970         unsigned long flags;
 971
 972         spin_lock_irqsave(&adev->smc_idx_lock, flags);
 973         WREG32(SMC_IND_INDEX_0, (reg));
 974         WREG32(SMC_IND_DATA_0, (v));
 975         spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
 976 }
 977
 978 static u32 si_uvd_ctx_rreg(struct amdgpu_device *adev, u32 reg)
 979 {
 980         unsigned long flags;
 981         u32 r;
 982
 983         spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
 984         WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
 985         r = RREG32(mmUVD_CTX_DATA);
 986         spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
 987         return r;
 988 }
 989
 990 static void si_uvd_ctx_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 991 {
 992         unsigned long flags;
 993
 994         spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
 995         WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
 996         WREG32(mmUVD_CTX_DATA, (v));
 997         spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
 998 }
 999
1000 static struct amdgpu_allowed_register_entry si_allowed_read_registers[] = {
1001         {GRBM_STATUS},
1002         {mmGRBM_STATUS2},
1003         {mmGRBM_STATUS_SE0},
1004         {mmGRBM_STATUS_SE1},
1005         {mmSRBM_STATUS},
1006         {mmSRBM_STATUS2},
1007         {DMA_STATUS_REG + DMA0_REGISTER_OFFSET},
1008         {DMA_STATUS_REG + DMA1_REGISTER_OFFSET},
1009         {mmCP_STAT},
1010         {mmCP_STALLED_STAT1},
1011         {mmCP_STALLED_STAT2},
1012         {mmCP_STALLED_STAT3},
1013         {GB_ADDR_CONFIG},
1014         {MC_ARB_RAMCFG},
1015         {GB_TILE_MODE0},
1016         {GB_TILE_MODE1},
1017         {GB_TILE_MODE2},
1018         {GB_TILE_MODE3},
1019         {GB_TILE_MODE4},
1020         {GB_TILE_MODE5},
1021         {GB_TILE_MODE6},
1022         {GB_TILE_MODE7},
1023         {GB_TILE_MODE8},
1024         {GB_TILE_MODE9},
1025         {GB_TILE_MODE10},
1026         {GB_TILE_MODE11},
1027         {GB_TILE_MODE12},
1028         {GB_TILE_MODE13},
1029         {GB_TILE_MODE14},
1030         {GB_TILE_MODE15},
1031         {GB_TILE_MODE16},
1032         {GB_TILE_MODE17},
1033         {GB_TILE_MODE18},
1034         {GB_TILE_MODE19},
1035         {GB_TILE_MODE20},
1036         {GB_TILE_MODE21},
1037         {GB_TILE_MODE22},
1038         {GB_TILE_MODE23},
1039         {GB_TILE_MODE24},
1040         {GB_TILE_MODE25},
1041         {GB_TILE_MODE26},
1042         {GB_TILE_MODE27},
1043         {GB_TILE_MODE28},
1044         {GB_TILE_MODE29},
1045         {GB_TILE_MODE30},
1046         {GB_TILE_MODE31},
1047         {CC_RB_BACKEND_DISABLE, true},
1048         {GC_USER_RB_BACKEND_DISABLE, true},
1049         {PA_SC_RASTER_CONFIG, true},
1050 };
1051
1052 static uint32_t si_get_register_value(struct amdgpu_device *adev,
1053                                       bool indexed, u32 se_num,
1054                                       u32 sh_num, u32 reg_offset)
1055 {
1056         if (indexed) {
1057                 uint32_t val;
1058                 unsigned se_idx = (se_num == 0xffffffff) ? 0 : se_num;
1059                 unsigned sh_idx = (sh_num == 0xffffffff) ? 0 : sh_num;
1060
1061                 switch (reg_offset) {
1062                 case mmCC_RB_BACKEND_DISABLE:
1063                         return adev->gfx.config.rb_config[se_idx][sh_idx].rb_backend_disable;
1064                 case mmGC_USER_RB_BACKEND_DISABLE:
1065                         return adev->gfx.config.rb_config[se_idx][sh_idx].user_rb_backend_disable;
1066                 case mmPA_SC_RASTER_CONFIG:
1067                         return adev->gfx.config.rb_config[se_idx][sh_idx].raster_config;
1068                 }
1069
1070                 mutex_lock(&adev->grbm_idx_mutex);
1071                 if (se_num != 0xffffffff || sh_num != 0xffffffff)
1072                         amdgpu_gfx_select_se_sh(adev, se_num, sh_num, 0xffffffff);
1073
1074                 val = RREG32(reg_offset);
1075
1076                 if (se_num != 0xffffffff || sh_num != 0xffffffff)
1077                         amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
1078                 mutex_unlock(&adev->grbm_idx_mutex);
1079                 return val;
1080         } else {
1081                 unsigned idx;
1082
1083                 switch (reg_offset) {
1084                 case mmGB_ADDR_CONFIG:
1085                         return adev->gfx.config.gb_addr_config;
1086                 case mmMC_ARB_RAMCFG:
1087                         return adev->gfx.config.mc_arb_ramcfg;
1088                 case mmGB_TILE_MODE0:
1089                 case mmGB_TILE_MODE1:
1090                 case mmGB_TILE_MODE2:
1091                 case mmGB_TILE_MODE3:
1092                 case mmGB_TILE_MODE4:
1093                 case mmGB_TILE_MODE5:
1094                 case mmGB_TILE_MODE6:
1095                 case mmGB_TILE_MODE7:
1096                 case mmGB_TILE_MODE8:
1097                 case mmGB_TILE_MODE9:
1098                 case mmGB_TILE_MODE10:
1099                 case mmGB_TILE_MODE11:
1100                 case mmGB_TILE_MODE12:
1101                 case mmGB_TILE_MODE13:
1102                 case mmGB_TILE_MODE14:
1103                 case mmGB_TILE_MODE15:
1104                 case mmGB_TILE_MODE16:
1105                 case mmGB_TILE_MODE17:
1106                 case mmGB_TILE_MODE18:
1107                 case mmGB_TILE_MODE19:
1108                 case mmGB_TILE_MODE20:
1109                 case mmGB_TILE_MODE21:
1110                 case mmGB_TILE_MODE22:
1111                 case mmGB_TILE_MODE23:
1112                 case mmGB_TILE_MODE24:
1113                 case mmGB_TILE_MODE25:
1114                 case mmGB_TILE_MODE26:
1115                 case mmGB_TILE_MODE27:
1116                 case mmGB_TILE_MODE28:
1117                 case mmGB_TILE_MODE29:
1118                 case mmGB_TILE_MODE30:
1119                 case mmGB_TILE_MODE31:
1120                         idx = (reg_offset - mmGB_TILE_MODE0);
1121                         return adev->gfx.config.tile_mode_array[idx];
1122                 default:
1123                         return RREG32(reg_offset);
1124                 }
1125         }
1126 }
1127 static int si_read_register(struct amdgpu_device *adev, u32 se_num,
1128                              u32 sh_num, u32 reg_offset, u32 *value)
1129 {
1130         uint32_t i;
1131
1132         *value = 0;
1133         for (i = 0; i < ARRAY_SIZE(si_allowed_read_registers); i++) {
1134                 bool indexed = si_allowed_read_registers[i].grbm_indexed;
1135
1136                 if (reg_offset != si_allowed_read_registers[i].reg_offset)
1137                         continue;
1138
1139                 *value = si_get_register_value(adev, indexed, se_num, sh_num,
1140                                                reg_offset);
1141                 return 0;
1142         }
1143         return -EINVAL;
1144 }
1145
1146 static bool si_read_disabled_bios(struct amdgpu_device *adev)
1147 {
1148         u32 bus_cntl;
1149         u32 d1vga_control = 0;
1150         u32 d2vga_control = 0;
1151         u32 vga_render_control = 0;
1152         u32 rom_cntl;
1153         bool r;
1154
1155         bus_cntl = RREG32(R600_BUS_CNTL);
1156         if (adev->mode_info.num_crtc) {
1157                 d1vga_control = RREG32(AVIVO_D1VGA_CONTROL);
1158                 d2vga_control = RREG32(AVIVO_D2VGA_CONTROL);
1159                 vga_render_control = RREG32(VGA_RENDER_CONTROL);
1160         }
1161         rom_cntl = RREG32(R600_ROM_CNTL);
1162
1163         /* enable the rom */
1164         WREG32(R600_BUS_CNTL, (bus_cntl & ~R600_BIOS_ROM_DIS));
1165         if (adev->mode_info.num_crtc) {
1166                 /* Disable VGA mode */
1167                 WREG32(AVIVO_D1VGA_CONTROL,
1168                        (d1vga_control & ~(AVIVO_DVGA_CONTROL_MODE_ENABLE |
1169                                           AVIVO_DVGA_CONTROL_TIMING_SELECT)));
1170                 WREG32(AVIVO_D2VGA_CONTROL,
1171                        (d2vga_control & ~(AVIVO_DVGA_CONTROL_MODE_ENABLE |
1172                                           AVIVO_DVGA_CONTROL_TIMING_SELECT)));
1173                 WREG32(VGA_RENDER_CONTROL,
1174                        (vga_render_control & C_000300_VGA_VSTATUS_CNTL));
1175         }
1176         WREG32(R600_ROM_CNTL, rom_cntl | R600_SCK_OVERWRITE);
1177
1178         r = amdgpu_read_bios(adev);
1179
1180         /* restore regs */
1181         WREG32(R600_BUS_CNTL, bus_cntl);
1182         if (adev->mode_info.num_crtc) {
1183                 WREG32(AVIVO_D1VGA_CONTROL, d1vga_control);
1184                 WREG32(AVIVO_D2VGA_CONTROL, d2vga_control);
1185                 WREG32(VGA_RENDER_CONTROL, vga_render_control);
1186         }
1187         WREG32(R600_ROM_CNTL, rom_cntl);
1188         return r;
1189 }
1190
1191 #define mmROM_INDEX 0x2A
1192 #define mmROM_DATA  0x2B
1193
1194 static bool si_read_bios_from_rom(struct amdgpu_device *adev,
1195                                   u8 *bios, u32 length_bytes)
1196 {
1197         u32 *dw_ptr;
1198         u32 i, length_dw;
1199
1200         if (bios == NULL)
1201                 return false;
1202         if (length_bytes == 0)
1203                 return false;
1204         /* APU vbios image is part of sbios image */
1205         if (adev->flags & AMD_IS_APU)
1206                 return false;
1207
1208         dw_ptr = (u32 *)bios;
1209         length_dw = ALIGN(length_bytes, 4) / 4;
1210         /* set rom index to 0 */
1211         WREG32(mmROM_INDEX, 0);
1212         for (i = 0; i < length_dw; i++)
1213                 dw_ptr[i] = RREG32(mmROM_DATA);
1214
1215         return true;
1216 }
1217
1218 //xxx: not implemented
1219 static int si_asic_reset(struct amdgpu_device *adev)
1220 {
1221         return 0;
1222 }
1223
1224 static bool si_asic_supports_baco(struct amdgpu_device *adev)
1225 {
1226         return false;
1227 }
1228
1229 static enum amd_reset_method
1230 si_asic_reset_method(struct amdgpu_device *adev)
1231 {
1232         if (amdgpu_reset_method != AMD_RESET_METHOD_LEGACY &&
1233             amdgpu_reset_method != -1)
1234                 dev_warn(adev->dev, "Specified reset method:%d isn't supported, using AUTO instead.\n",
1235                                   amdgpu_reset_method);
1236
1237         return AMD_RESET_METHOD_LEGACY;
1238 }
1239
1240 static u32 si_get_config_memsize(struct amdgpu_device *adev)
1241 {
1242         return RREG32(mmCONFIG_MEMSIZE);
1243 }
1244
1245 static void si_vga_set_state(struct amdgpu_device *adev, bool state)
1246 {
1247         uint32_t temp;
1248
1249         temp = RREG32(CONFIG_CNTL);
1250         if (state == false) {
1251                 temp &= ~(1<<0);
1252                 temp |= (1<<1);
1253         } else {
1254                 temp &= ~(1<<1);
1255         }
1256         WREG32(CONFIG_CNTL, temp);
1257 }
1258
1259 static u32 si_get_xclk(struct amdgpu_device *adev)
1260 {
1261         u32 reference_clock = adev->clock.spll.reference_freq;
1262         u32 tmp;
1263
1264         tmp = RREG32(CG_CLKPIN_CNTL_2);
1265         if (tmp & MUX_TCLK_TO_XCLK)
1266                 return TCLK;
1267
1268         tmp = RREG32(CG_CLKPIN_CNTL);
1269         if (tmp & XTALIN_DIVIDE)
1270                 return reference_clock / 4;
1271
1272         return reference_clock;
1273 }
1274
1275 static void si_flush_hdp(struct amdgpu_device *adev, struct amdgpu_ring *ring)
1276 {
1277         if (!ring || !ring->funcs->emit_wreg) {
1278                 WREG32(mmHDP_MEM_COHERENCY_FLUSH_CNTL, 1);
1279                 RREG32(mmHDP_MEM_COHERENCY_FLUSH_CNTL);
1280         } else {
1281                 amdgpu_ring_emit_wreg(ring, mmHDP_MEM_COHERENCY_FLUSH_CNTL, 1);
1282         }
1283 }
1284
1285 static void si_invalidate_hdp(struct amdgpu_device *adev,
1286                               struct amdgpu_ring *ring)
1287 {
1288         if (!ring || !ring->funcs->emit_wreg) {
1289                 WREG32(mmHDP_DEBUG0, 1);
1290                 RREG32(mmHDP_DEBUG0);
1291         } else {
1292                 amdgpu_ring_emit_wreg(ring, mmHDP_DEBUG0, 1);
1293         }
1294 }
1295
1296 static bool si_need_full_reset(struct amdgpu_device *adev)
1297 {
1298         /* change this when we support soft reset */
1299         return true;
1300 }
1301
1302 static bool si_need_reset_on_init(struct amdgpu_device *adev)
1303 {
1304         return false;
1305 }
1306
1307 static int si_get_pcie_lanes(struct amdgpu_device *adev)
1308 {
1309         u32 link_width_cntl;
1310
1311         if (adev->flags & AMD_IS_APU)
1312                 return 0;
1313
1314         link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
1315
1316         switch ((link_width_cntl & LC_LINK_WIDTH_RD_MASK) >> LC_LINK_WIDTH_RD_SHIFT) {
1317         case LC_LINK_WIDTH_X1:
1318                 return 1;
1319         case LC_LINK_WIDTH_X2:
1320                 return 2;
1321         case LC_LINK_WIDTH_X4:
1322                 return 4;
1323         case LC_LINK_WIDTH_X8:
1324                 return 8;
1325         case LC_LINK_WIDTH_X0:
1326         case LC_LINK_WIDTH_X16:
1327         default:
1328                 return 16;
1329         }
1330 }
1331
1332 static void si_set_pcie_lanes(struct amdgpu_device *adev, int lanes)
1333 {
1334         u32 link_width_cntl, mask;
1335
1336         if (adev->flags & AMD_IS_APU)
1337                 return;
1338
1339         switch (lanes) {
1340         case 0:
1341                 mask = LC_LINK_WIDTH_X0;
1342                 break;
1343         case 1:
1344                 mask = LC_LINK_WIDTH_X1;
1345                 break;
1346         case 2:
1347                 mask = LC_LINK_WIDTH_X2;
1348                 break;
1349         case 4:
1350                 mask = LC_LINK_WIDTH_X4;
1351                 break;
1352         case 8:
1353                 mask = LC_LINK_WIDTH_X8;
1354                 break;
1355         case 16:
1356                 mask = LC_LINK_WIDTH_X16;
1357                 break;
1358         default:
1359                 DRM_ERROR("invalid pcie lane request: %d\n", lanes);
1360                 return;
1361         }
1362
1363         link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
1364         link_width_cntl &= ~LC_LINK_WIDTH_MASK;
1365         link_width_cntl |= mask << LC_LINK_WIDTH_SHIFT;
1366         link_width_cntl |= (LC_RECONFIG_NOW |
1367                             LC_RECONFIG_ARC_MISSING_ESCAPE);
1368
1369         WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
1370 }
1371
1372 static void si_get_pcie_usage(struct amdgpu_device *adev, uint64_t *count0,
1373                               uint64_t *count1)
1374 {
1375         uint32_t perfctr = 0;
1376         uint64_t cnt0_of, cnt1_of;
1377         int tmp;
1378
1379         /* This reports 0 on APUs, so return to avoid writing/reading registers
1380          * that may or may not be different from their GPU counterparts
1381          */
1382         if (adev->flags & AMD_IS_APU)
1383                 return;
1384
1385         /* Set the 2 events that we wish to watch, defined above */
1386         /* Reg 40 is # received msgs, Reg 104 is # of posted requests sent */
1387         perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK, EVENT0_SEL, 40);
1388         perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK, EVENT1_SEL, 104);
1389
1390         /* Write to enable desired perf counters */
1391         WREG32_PCIE(ixPCIE_PERF_CNTL_TXCLK, perfctr);
1392         /* Zero out and enable the perf counters
1393          * Write 0x5:
1394          * Bit 0 = Start all counters(1)
1395          * Bit 2 = Global counter reset enable(1)
1396          */
1397         WREG32_PCIE(ixPCIE_PERF_COUNT_CNTL, 0x00000005);
1398
1399         msleep(1000);
1400
1401         /* Load the shadow and disable the perf counters
1402          * Write 0x2:
1403          * Bit 0 = Stop counters(0)
1404          * Bit 1 = Load the shadow counters(1)
1405          */
1406         WREG32_PCIE(ixPCIE_PERF_COUNT_CNTL, 0x00000002);
1407
1408         /* Read register values to get any >32bit overflow */
1409         tmp = RREG32_PCIE(ixPCIE_PERF_CNTL_TXCLK);
1410         cnt0_of = REG_GET_FIELD(tmp, PCIE_PERF_CNTL_TXCLK, COUNTER0_UPPER);
1411         cnt1_of = REG_GET_FIELD(tmp, PCIE_PERF_CNTL_TXCLK, COUNTER1_UPPER);
1412
1413         /* Get the values and add the overflow */
1414         *count0 = RREG32_PCIE(ixPCIE_PERF_COUNT0_TXCLK) | (cnt0_of << 32);
1415         *count1 = RREG32_PCIE(ixPCIE_PERF_COUNT1_TXCLK) | (cnt1_of << 32);
1416 }
1417
1418 static uint64_t si_get_pcie_replay_count(struct amdgpu_device *adev)
1419 {
1420         uint64_t nak_r, nak_g;
1421
1422         /* Get the number of NAKs received and generated */
1423         nak_r = RREG32_PCIE(ixPCIE_RX_NUM_NAK);
1424         nak_g = RREG32_PCIE(ixPCIE_RX_NUM_NAK_GENERATED);
1425
1426         /* Add the total number of NAKs, i.e the number of replays */
1427         return (nak_r + nak_g);
1428 }
1429
1430 static int si_uvd_send_upll_ctlreq(struct amdgpu_device *adev,
1431                                    unsigned cg_upll_func_cntl)
1432 {
1433         unsigned i;
1434
1435         /* Make sure UPLL_CTLREQ is deasserted */
1436         WREG32_P(cg_upll_func_cntl, 0, ~UPLL_CTLREQ_MASK);
1437
1438         mdelay(10);
1439
1440         /* Assert UPLL_CTLREQ */
1441         WREG32_P(cg_upll_func_cntl, UPLL_CTLREQ_MASK, ~UPLL_CTLREQ_MASK);
1442
1443         /* Wait for CTLACK and CTLACK2 to get asserted */
1444         for (i = 0; i < SI_MAX_CTLACKS_ASSERTION_WAIT; ++i) {
1445                 uint32_t mask = UPLL_CTLACK_MASK | UPLL_CTLACK2_MASK;
1446
1447                 if ((RREG32(cg_upll_func_cntl) & mask) == mask)
1448                         break;
1449                 mdelay(10);
1450         }
1451
1452         /* Deassert UPLL_CTLREQ */
1453         WREG32_P(cg_upll_func_cntl, 0, ~UPLL_CTLREQ_MASK);
1454
1455         if (i == SI_MAX_CTLACKS_ASSERTION_WAIT) {
1456                 DRM_ERROR("Timeout setting UVD clocks!\n");
1457                 return -ETIMEDOUT;
1458         }
1459
1460         return 0;
1461 }
1462
1463 static unsigned si_uvd_calc_upll_post_div(unsigned vco_freq,
1464                                           unsigned target_freq,
1465                                           unsigned pd_min,
1466                                           unsigned pd_even)
1467 {
1468         unsigned post_div = vco_freq / target_freq;
1469
1470         /* Adjust to post divider minimum value */
1471         if (post_div < pd_min)
1472                 post_div = pd_min;
1473
1474         /* We alway need a frequency less than or equal the target */
1475         if ((vco_freq / post_div) > target_freq)
1476                 post_div += 1;
1477
1478         /* Post dividers above a certain value must be even */
1479         if (post_div > pd_even && post_div % 2)
1480                 post_div += 1;
1481
1482         return post_div;
1483 }
1484
1485 /**
1486  * si_calc_upll_dividers - calc UPLL clock dividers
1487  *
1488  * @adev: amdgpu_device pointer
1489  * @vclk: wanted VCLK
1490  * @dclk: wanted DCLK
1491  * @vco_min: minimum VCO frequency
1492  * @vco_max: maximum VCO frequency
1493  * @fb_factor: factor to multiply vco freq with
1494  * @fb_mask: limit and bitmask for feedback divider
1495  * @pd_min: post divider minimum
1496  * @pd_max: post divider maximum
1497  * @pd_even: post divider must be even above this value
1498  * @optimal_fb_div: resulting feedback divider
1499  * @optimal_vclk_div: resulting vclk post divider
1500  * @optimal_dclk_div: resulting dclk post divider
1501  *
1502  * Calculate dividers for UVDs UPLL (except APUs).
1503  * Returns zero on success; -EINVAL on error.
1504  */
1505 static int si_calc_upll_dividers(struct amdgpu_device *adev,
1506                                  unsigned vclk, unsigned dclk,
1507                                  unsigned vco_min, unsigned vco_max,
1508                                  unsigned fb_factor, unsigned fb_mask,
1509                                  unsigned pd_min, unsigned pd_max,
1510                                  unsigned pd_even,
1511                                  unsigned *optimal_fb_div,
1512                                  unsigned *optimal_vclk_div,
1513                                  unsigned *optimal_dclk_div)
1514 {
1515         unsigned vco_freq, ref_freq = adev->clock.spll.reference_freq;
1516
1517         /* Start off with something large */
1518         unsigned optimal_score = ~0;
1519
1520         /* Loop through vco from low to high */
1521         vco_min = max(max(vco_min, vclk), dclk);
1522         for (vco_freq = vco_min; vco_freq <= vco_max; vco_freq += 100) {
1523                 uint64_t fb_div = (uint64_t)vco_freq * fb_factor;
1524                 unsigned vclk_div, dclk_div, score;
1525
1526                 do_div(fb_div, ref_freq);
1527
1528                 /* fb div out of range ? */
1529                 if (fb_div > fb_mask)
1530                         break; /* It can oly get worse */
1531
1532                 fb_div &= fb_mask;
1533
1534                 /* Calc vclk divider with current vco freq */
1535                 vclk_div = si_uvd_calc_upll_post_div(vco_freq, vclk,
1536                                                      pd_min, pd_even);
1537                 if (vclk_div > pd_max)
1538                         break; /* vco is too big, it has to stop */
1539
1540                 /* Calc dclk divider with current vco freq */
1541                 dclk_div = si_uvd_calc_upll_post_div(vco_freq, dclk,
1542                                                      pd_min, pd_even);
1543                 if (dclk_div > pd_max)
1544                         break; /* vco is too big, it has to stop */
1545
1546                 /* Calc score with current vco freq */
1547                 score = vclk - (vco_freq / vclk_div) + dclk - (vco_freq / dclk_div);
1548
1549                 /* Determine if this vco setting is better than current optimal settings */
1550                 if (score < optimal_score) {
1551                         *optimal_fb_div = fb_div;
1552                         *optimal_vclk_div = vclk_div;
1553                         *optimal_dclk_div = dclk_div;
1554                         optimal_score = score;
1555                         if (optimal_score == 0)
1556                                 break; /* It can't get better than this */
1557                 }
1558         }
1559
1560         /* Did we found a valid setup ? */
1561         if (optimal_score == ~0)
1562                 return -EINVAL;
1563
1564         return 0;
1565 }
1566
1567 static int si_set_uvd_clocks(struct amdgpu_device *adev, u32 vclk, u32 dclk)
1568 {
1569         unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
1570         int r;
1571
1572         /* Bypass vclk and dclk with bclk */
1573         WREG32_P(CG_UPLL_FUNC_CNTL_2,
1574                  VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
1575                  ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1576
1577         /* Put PLL in bypass mode */
1578         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
1579
1580         if (!vclk || !dclk) {
1581                 /* Keep the Bypass mode */
1582                 return 0;
1583         }
1584
1585         r = si_calc_upll_dividers(adev, vclk, dclk, 125000, 250000,
1586                                   16384, 0x03FFFFFF, 0, 128, 5,
1587                                   &fb_div, &vclk_div, &dclk_div);
1588         if (r)
1589                 return r;
1590
1591         /* Set RESET_ANTI_MUX to 0 */
1592         WREG32_P(CG_UPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);
1593
1594         /* Set VCO_MODE to 1 */
1595         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
1596
1597         /* Disable sleep mode */
1598         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
1599
1600         /* Deassert UPLL_RESET */
1601         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1602
1603         mdelay(1);
1604
1605         r = si_uvd_send_upll_ctlreq(adev, CG_UPLL_FUNC_CNTL);
1606         if (r)
1607                 return r;
1608
1609         /* Assert UPLL_RESET again */
1610         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
1611
1612         /* Disable spread spectrum. */
1613         WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
1614
1615         /* Set feedback divider */
1616         WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
1617
1618         /* Set ref divider to 0 */
1619         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
1620
1621         if (fb_div < 307200)
1622                 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
1623         else
1624                 WREG32_P(CG_UPLL_FUNC_CNTL_4,
1625                          UPLL_SPARE_ISPARE9,
1626                          ~UPLL_SPARE_ISPARE9);
1627
1628         /* Set PDIV_A and PDIV_B */
1629         WREG32_P(CG_UPLL_FUNC_CNTL_2,
1630                  UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
1631                  ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
1632
1633         /* Give the PLL some time to settle */
1634         mdelay(15);
1635
1636         /* Deassert PLL_RESET */
1637         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1638
1639         mdelay(15);
1640
1641         /* Switch from bypass mode to normal mode */
1642         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
1643
1644         r = si_uvd_send_upll_ctlreq(adev, CG_UPLL_FUNC_CNTL);
1645         if (r)
1646                 return r;
1647
1648         /* Switch VCLK and DCLK selection */
1649         WREG32_P(CG_UPLL_FUNC_CNTL_2,
1650                  VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
1651                  ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1652
1653         mdelay(100);
1654
1655         return 0;
1656 }
1657
1658 static int si_vce_send_vcepll_ctlreq(struct amdgpu_device *adev)
1659 {
1660         unsigned i;
1661
1662         /* Make sure VCEPLL_CTLREQ is deasserted */
1663         WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~UPLL_CTLREQ_MASK);
1664
1665         mdelay(10);
1666
1667         /* Assert UPLL_CTLREQ */
1668         WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, UPLL_CTLREQ_MASK, ~UPLL_CTLREQ_MASK);
1669
1670         /* Wait for CTLACK and CTLACK2 to get asserted */
1671         for (i = 0; i < SI_MAX_CTLACKS_ASSERTION_WAIT; ++i) {
1672                 uint32_t mask = UPLL_CTLACK_MASK | UPLL_CTLACK2_MASK;
1673
1674                 if ((RREG32_SMC(CG_VCEPLL_FUNC_CNTL) & mask) == mask)
1675                         break;
1676                 mdelay(10);
1677         }
1678
1679         /* Deassert UPLL_CTLREQ */
1680         WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~UPLL_CTLREQ_MASK);
1681
1682         if (i == SI_MAX_CTLACKS_ASSERTION_WAIT) {
1683                 DRM_ERROR("Timeout setting UVD clocks!\n");
1684                 return -ETIMEDOUT;
1685         }
1686
1687         return 0;
1688 }
1689
1690 static int si_set_vce_clocks(struct amdgpu_device *adev, u32 evclk, u32 ecclk)
1691 {
1692         unsigned fb_div = 0, evclk_div = 0, ecclk_div = 0;
1693         int r;
1694
1695         /* Bypass evclk and ecclk with bclk */
1696         WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_2,
1697                      EVCLK_SRC_SEL(1) | ECCLK_SRC_SEL(1),
1698                      ~(EVCLK_SRC_SEL_MASK | ECCLK_SRC_SEL_MASK));
1699
1700         /* Put PLL in bypass mode */
1701         WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_BYPASS_EN_MASK,
1702                      ~VCEPLL_BYPASS_EN_MASK);
1703
1704         if (!evclk || !ecclk) {
1705                 /* Keep the Bypass mode, put PLL to sleep */
1706                 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_SLEEP_MASK,
1707                              ~VCEPLL_SLEEP_MASK);
1708                 return 0;
1709         }
1710
1711         r = si_calc_upll_dividers(adev, evclk, ecclk, 125000, 250000,
1712                                   16384, 0x03FFFFFF, 0, 128, 5,
1713                                   &fb_div, &evclk_div, &ecclk_div);
1714         if (r)
1715                 return r;
1716
1717         /* Set RESET_ANTI_MUX to 0 */
1718         WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);
1719
1720         /* Set VCO_MODE to 1 */
1721         WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_VCO_MODE_MASK,
1722                      ~VCEPLL_VCO_MODE_MASK);
1723
1724         /* Toggle VCEPLL_SLEEP to 1 then back to 0 */
1725         WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_SLEEP_MASK,
1726                      ~VCEPLL_SLEEP_MASK);
1727         WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_SLEEP_MASK);
1728
1729         /* Deassert VCEPLL_RESET */
1730         WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_RESET_MASK);
1731
1732         mdelay(1);
1733
1734         r = si_vce_send_vcepll_ctlreq(adev);
1735         if (r)
1736                 return r;
1737
1738         /* Assert VCEPLL_RESET again */
1739         WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_RESET_MASK, ~VCEPLL_RESET_MASK);
1740
1741         /* Disable spread spectrum. */
1742         WREG32_SMC_P(CG_VCEPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
1743
1744         /* Set feedback divider */
1745         WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_3,
1746                      VCEPLL_FB_DIV(fb_div),
1747                      ~VCEPLL_FB_DIV_MASK);
1748
1749         /* Set ref divider to 0 */
1750         WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_REF_DIV_MASK);
1751
1752         /* Set PDIV_A and PDIV_B */
1753         WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_2,
1754                      VCEPLL_PDIV_A(evclk_div) | VCEPLL_PDIV_B(ecclk_div),
1755                      ~(VCEPLL_PDIV_A_MASK | VCEPLL_PDIV_B_MASK));
1756
1757         /* Give the PLL some time to settle */
1758         mdelay(15);
1759
1760         /* Deassert PLL_RESET */
1761         WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_RESET_MASK);
1762
1763         mdelay(15);
1764
1765         /* Switch from bypass mode to normal mode */
1766         WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_BYPASS_EN_MASK);
1767
1768         r = si_vce_send_vcepll_ctlreq(adev);
1769         if (r)
1770                 return r;
1771
1772         /* Switch VCLK and DCLK selection */
1773         WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_2,
1774                      EVCLK_SRC_SEL(16) | ECCLK_SRC_SEL(16),
1775                      ~(EVCLK_SRC_SEL_MASK | ECCLK_SRC_SEL_MASK));
1776
1777         mdelay(100);
1778
1779         return 0;
1780 }
1781
1782 static const struct amdgpu_asic_funcs si_asic_funcs =
1783 {
1784         .read_disabled_bios = &si_read_disabled_bios,
1785         .read_bios_from_rom = &si_read_bios_from_rom,
1786         .read_register = &si_read_register,
1787         .reset = &si_asic_reset,
1788         .reset_method = &si_asic_reset_method,
1789         .set_vga_state = &si_vga_set_state,
1790         .get_xclk = &si_get_xclk,
1791         .set_uvd_clocks = &si_set_uvd_clocks,
1792         .set_vce_clocks = &si_set_vce_clocks,
1793         .get_pcie_lanes = &si_get_pcie_lanes,
1794         .set_pcie_lanes = &si_set_pcie_lanes,
1795         .get_config_memsize = &si_get_config_memsize,
1796         .flush_hdp = &si_flush_hdp,
1797         .invalidate_hdp = &si_invalidate_hdp,
1798         .need_full_reset = &si_need_full_reset,
1799         .get_pcie_usage = &si_get_pcie_usage,
1800         .need_reset_on_init = &si_need_reset_on_init,
1801         .get_pcie_replay_count = &si_get_pcie_replay_count,
1802         .supports_baco = &si_asic_supports_baco,
1803 };
1804
1805 static uint32_t si_get_rev_id(struct amdgpu_device *adev)
1806 {
1807         return (RREG32(CC_DRM_ID_STRAPS) & CC_DRM_ID_STRAPS__ATI_REV_ID_MASK)
1808                 >> CC_DRM_ID_STRAPS__ATI_REV_ID__SHIFT;
1809 }
1810
1811 static int si_common_early_init(void *handle)
1812 {
1813         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1814
1815         adev->smc_rreg = &si_smc_rreg;
1816         adev->smc_wreg = &si_smc_wreg;
1817         adev->pcie_rreg = &si_pcie_rreg;
1818         adev->pcie_wreg = &si_pcie_wreg;
1819         adev->pciep_rreg = &si_pciep_rreg;
1820         adev->pciep_wreg = &si_pciep_wreg;
1821         adev->uvd_ctx_rreg = si_uvd_ctx_rreg;
1822         adev->uvd_ctx_wreg = si_uvd_ctx_wreg;
1823         adev->didt_rreg = NULL;
1824         adev->didt_wreg = NULL;
1825
1826         adev->asic_funcs = &si_asic_funcs;
1827
1828         adev->rev_id = si_get_rev_id(adev);
1829         adev->external_rev_id = 0xFF;
1830         switch (adev->asic_type) {
1831         case CHIP_TAHITI:
1832                 adev->cg_flags =
1833                         AMD_CG_SUPPORT_GFX_MGCG |
1834                         AMD_CG_SUPPORT_GFX_MGLS |
1835                         /*AMD_CG_SUPPORT_GFX_CGCG |*/
1836                         AMD_CG_SUPPORT_GFX_CGLS |
1837                         AMD_CG_SUPPORT_GFX_CGTS |
1838                         AMD_CG_SUPPORT_GFX_CP_LS |
1839                         AMD_CG_SUPPORT_MC_MGCG |
1840                         AMD_CG_SUPPORT_SDMA_MGCG |
1841                         AMD_CG_SUPPORT_BIF_LS |
1842                         AMD_CG_SUPPORT_VCE_MGCG |
1843                         AMD_CG_SUPPORT_UVD_MGCG |
1844                         AMD_CG_SUPPORT_HDP_LS |
1845                         AMD_CG_SUPPORT_HDP_MGCG;
1846                 adev->pg_flags = 0;
1847                 adev->external_rev_id = (adev->rev_id == 0) ? 1 :
1848                                         (adev->rev_id == 1) ? 5 : 6;
1849                 break;
1850         case CHIP_PITCAIRN:
1851                 adev->cg_flags =
1852                         AMD_CG_SUPPORT_GFX_MGCG |
1853                         AMD_CG_SUPPORT_GFX_MGLS |
1854                         /*AMD_CG_SUPPORT_GFX_CGCG |*/
1855                         AMD_CG_SUPPORT_GFX_CGLS |
1856                         AMD_CG_SUPPORT_GFX_CGTS |
1857                         AMD_CG_SUPPORT_GFX_CP_LS |
1858                         AMD_CG_SUPPORT_GFX_RLC_LS |
1859                         AMD_CG_SUPPORT_MC_LS |
1860                         AMD_CG_SUPPORT_MC_MGCG |
1861                         AMD_CG_SUPPORT_SDMA_MGCG |
1862                         AMD_CG_SUPPORT_BIF_LS |
1863                         AMD_CG_SUPPORT_VCE_MGCG |
1864                         AMD_CG_SUPPORT_UVD_MGCG |
1865                         AMD_CG_SUPPORT_HDP_LS |
1866                         AMD_CG_SUPPORT_HDP_MGCG;
1867                 adev->pg_flags = 0;
1868                 adev->external_rev_id = adev->rev_id + 20;
1869                 break;
1870
1871         case CHIP_VERDE:
1872                 adev->cg_flags =
1873                         AMD_CG_SUPPORT_GFX_MGCG |
1874                         AMD_CG_SUPPORT_GFX_MGLS |
1875                         AMD_CG_SUPPORT_GFX_CGLS |
1876                         AMD_CG_SUPPORT_GFX_CGTS |
1877                         AMD_CG_SUPPORT_GFX_CGTS_LS |
1878                         AMD_CG_SUPPORT_GFX_CP_LS |
1879                         AMD_CG_SUPPORT_MC_LS |
1880                         AMD_CG_SUPPORT_MC_MGCG |
1881                         AMD_CG_SUPPORT_SDMA_MGCG |
1882                         AMD_CG_SUPPORT_SDMA_LS |
1883                         AMD_CG_SUPPORT_BIF_LS |
1884                         AMD_CG_SUPPORT_VCE_MGCG |
1885                         AMD_CG_SUPPORT_UVD_MGCG |
1886                         AMD_CG_SUPPORT_HDP_LS |
1887                         AMD_CG_SUPPORT_HDP_MGCG;
1888                 adev->pg_flags = 0;
1889                 //???
1890                 adev->external_rev_id = adev->rev_id + 40;
1891                 break;
1892         case CHIP_OLAND:
1893                 adev->cg_flags =
1894                         AMD_CG_SUPPORT_GFX_MGCG |
1895                         AMD_CG_SUPPORT_GFX_MGLS |
1896                         /*AMD_CG_SUPPORT_GFX_CGCG |*/
1897                         AMD_CG_SUPPORT_GFX_CGLS |
1898                         AMD_CG_SUPPORT_GFX_CGTS |
1899                         AMD_CG_SUPPORT_GFX_CP_LS |
1900                         AMD_CG_SUPPORT_GFX_RLC_LS |
1901                         AMD_CG_SUPPORT_MC_LS |
1902                         AMD_CG_SUPPORT_MC_MGCG |
1903                         AMD_CG_SUPPORT_SDMA_MGCG |
1904                         AMD_CG_SUPPORT_BIF_LS |
1905                         AMD_CG_SUPPORT_UVD_MGCG |
1906                         AMD_CG_SUPPORT_HDP_LS |
1907                         AMD_CG_SUPPORT_HDP_MGCG;
1908                 adev->pg_flags = 0;
1909                 adev->external_rev_id = 60;
1910                 break;
1911         case CHIP_HAINAN:
1912                 adev->cg_flags =
1913                         AMD_CG_SUPPORT_GFX_MGCG |
1914                         AMD_CG_SUPPORT_GFX_MGLS |
1915                         /*AMD_CG_SUPPORT_GFX_CGCG |*/
1916                         AMD_CG_SUPPORT_GFX_CGLS |
1917                         AMD_CG_SUPPORT_GFX_CGTS |
1918                         AMD_CG_SUPPORT_GFX_CP_LS |
1919                         AMD_CG_SUPPORT_GFX_RLC_LS |
1920                         AMD_CG_SUPPORT_MC_LS |
1921                         AMD_CG_SUPPORT_MC_MGCG |
1922                         AMD_CG_SUPPORT_SDMA_MGCG |
1923                         AMD_CG_SUPPORT_BIF_LS |
1924                         AMD_CG_SUPPORT_HDP_LS |
1925                         AMD_CG_SUPPORT_HDP_MGCG;
1926                 adev->pg_flags = 0;
1927                 adev->external_rev_id = 70;
1928                 break;
1929
1930         default:
1931                 return -EINVAL;
1932         }
1933
1934         return 0;
1935 }
1936
1937 static int si_common_sw_init(void *handle)
1938 {
1939         return 0;
1940 }
1941
1942 static int si_common_sw_fini(void *handle)
1943 {
1944         return 0;
1945 }
1946
1947
1948 static void si_init_golden_registers(struct amdgpu_device *adev)
1949 {
1950         switch (adev->asic_type) {
1951         case CHIP_TAHITI:
1952                 amdgpu_device_program_register_sequence(adev,
1953                                                         tahiti_golden_registers,
1954                                                         ARRAY_SIZE(tahiti_golden_registers));
1955                 amdgpu_device_program_register_sequence(adev,
1956                                                         tahiti_golden_rlc_registers,
1957                                                         ARRAY_SIZE(tahiti_golden_rlc_registers));
1958                 amdgpu_device_program_register_sequence(adev,
1959                                                         tahiti_mgcg_cgcg_init,
1960                                                         ARRAY_SIZE(tahiti_mgcg_cgcg_init));
1961                 amdgpu_device_program_register_sequence(adev,
1962                                                         tahiti_golden_registers2,
1963                                                         ARRAY_SIZE(tahiti_golden_registers2));
1964                 break;
1965         case CHIP_PITCAIRN:
1966                 amdgpu_device_program_register_sequence(adev,
1967                                                         pitcairn_golden_registers,
1968                                                         ARRAY_SIZE(pitcairn_golden_registers));
1969                 amdgpu_device_program_register_sequence(adev,
1970                                                         pitcairn_golden_rlc_registers,
1971                                                         ARRAY_SIZE(pitcairn_golden_rlc_registers));
1972                 amdgpu_device_program_register_sequence(adev,
1973                                                         pitcairn_mgcg_cgcg_init,
1974                                                         ARRAY_SIZE(pitcairn_mgcg_cgcg_init));
1975                 break;
1976         case CHIP_VERDE:
1977                 amdgpu_device_program_register_sequence(adev,
1978                                                         verde_golden_registers,
1979                                                         ARRAY_SIZE(verde_golden_registers));
1980                 amdgpu_device_program_register_sequence(adev,
1981                                                         verde_golden_rlc_registers,
1982                                                         ARRAY_SIZE(verde_golden_rlc_registers));
1983                 amdgpu_device_program_register_sequence(adev,
1984                                                         verde_mgcg_cgcg_init,
1985                                                         ARRAY_SIZE(verde_mgcg_cgcg_init));
1986                 amdgpu_device_program_register_sequence(adev,
1987                                                         verde_pg_init,
1988                                                         ARRAY_SIZE(verde_pg_init));
1989                 break;
1990         case CHIP_OLAND:
1991                 amdgpu_device_program_register_sequence(adev,
1992                                                         oland_golden_registers,
1993                                                         ARRAY_SIZE(oland_golden_registers));
1994                 amdgpu_device_program_register_sequence(adev,
1995                                                         oland_golden_rlc_registers,
1996                                                         ARRAY_SIZE(oland_golden_rlc_registers));
1997                 amdgpu_device_program_register_sequence(adev,
1998                                                         oland_mgcg_cgcg_init,
1999                                                         ARRAY_SIZE(oland_mgcg_cgcg_init));
2000                 break;
2001         case CHIP_HAINAN:
2002                 amdgpu_device_program_register_sequence(adev,
2003                                                         hainan_golden_registers,
2004                                                         ARRAY_SIZE(hainan_golden_registers));
2005                 amdgpu_device_program_register_sequence(adev,
2006                                                         hainan_golden_registers2,
2007                                                         ARRAY_SIZE(hainan_golden_registers2));
2008                 amdgpu_device_program_register_sequence(adev,
2009                                                         hainan_mgcg_cgcg_init,
2010                                                         ARRAY_SIZE(hainan_mgcg_cgcg_init));
2011                 break;
2012
2013
2014         default:
2015                 BUG();
2016         }
2017 }
2018
2019 static void si_pcie_gen3_enable(struct amdgpu_device *adev)
2020 {
2021         struct pci_dev *root = adev->pdev->bus->self;
2022         u32 speed_cntl, current_data_rate;
2023         int i;
2024         u16 tmp16;
2025
2026         if (pci_is_root_bus(adev->pdev->bus))
2027                 return;
2028
2029         if (amdgpu_pcie_gen2 == 0)
2030                 return;
2031
2032         if (adev->flags & AMD_IS_APU)
2033                 return;
2034
2035         if (!(adev->pm.pcie_gen_mask & (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
2036                                         CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)))
2037                 return;
2038
2039         speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
2040         current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
2041                 LC_CURRENT_DATA_RATE_SHIFT;
2042         if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
2043                 if (current_data_rate == 2) {
2044                         DRM_INFO("PCIE gen 3 link speeds already enabled\n");
2045                         return;
2046                 }
2047                 DRM_INFO("enabling PCIE gen 3 link speeds, disable with amdgpu.pcie_gen2=0\n");
2048         } else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2) {
2049                 if (current_data_rate == 1) {
2050                         DRM_INFO("PCIE gen 2 link speeds already enabled\n");
2051                         return;
2052                 }
2053                 DRM_INFO("enabling PCIE gen 2 link speeds, disable with amdgpu.pcie_gen2=0\n");
2054         }
2055
2056         if (!pci_is_pcie(root) || !pci_is_pcie(adev->pdev))
2057                 return;
2058
2059         if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
2060                 if (current_data_rate != 2) {
2061                         u16 bridge_cfg, gpu_cfg;
2062                         u16 bridge_cfg2, gpu_cfg2;
2063                         u32 max_lw, current_lw, tmp;
2064
2065                         pcie_capability_read_word(root, PCI_EXP_LNKCTL,
2066                                                   &bridge_cfg);
2067                         pcie_capability_read_word(adev->pdev, PCI_EXP_LNKCTL,
2068                                                   &gpu_cfg);
2069
2070                         tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
2071                         pcie_capability_write_word(root, PCI_EXP_LNKCTL, tmp16);
2072
2073                         tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
2074                         pcie_capability_write_word(adev->pdev, PCI_EXP_LNKCTL,
2075                                                    tmp16);
2076
2077                         tmp = RREG32_PCIE(PCIE_LC_STATUS1);
2078                         max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
2079                         current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
2080
2081                         if (current_lw < max_lw) {
2082                                 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
2083                                 if (tmp & LC_RENEGOTIATION_SUPPORT) {
2084                                         tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
2085                                         tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
2086                                         tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
2087                                         WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
2088                                 }
2089                         }
2090
2091                         for (i = 0; i < 10; i++) {
2092                                 pcie_capability_read_word(adev->pdev,
2093                                                           PCI_EXP_DEVSTA,
2094                                                           &tmp16);
2095                                 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
2096                                         break;
2097
2098                                 pcie_capability_read_word(root, PCI_EXP_LNKCTL,
2099                                                           &bridge_cfg);
2100                                 pcie_capability_read_word(adev->pdev,
2101                                                           PCI_EXP_LNKCTL,
2102                                                           &gpu_cfg);
2103
2104                                 pcie_capability_read_word(root, PCI_EXP_LNKCTL2,
2105                                                           &bridge_cfg2);
2106                                 pcie_capability_read_word(adev->pdev,
2107                                                           PCI_EXP_LNKCTL2,
2108                                                           &gpu_cfg2);
2109
2110                                 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
2111                                 tmp |= LC_SET_QUIESCE;
2112                                 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
2113
2114                                 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
2115                                 tmp |= LC_REDO_EQ;
2116                                 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
2117
2118                                 mdelay(100);
2119
2120                                 pcie_capability_read_word(root, PCI_EXP_LNKCTL,
2121                                                           &tmp16);
2122                                 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
2123                                 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
2124                                 pcie_capability_write_word(root, PCI_EXP_LNKCTL,
2125                                                            tmp16);
2126
2127                                 pcie_capability_read_word(adev->pdev,
2128                                                           PCI_EXP_LNKCTL,
2129                                                           &tmp16);
2130                                 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
2131                                 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
2132                                 pcie_capability_write_word(adev->pdev,
2133                                                            PCI_EXP_LNKCTL,
2134                                                            tmp16);
2135
2136                                 pcie_capability_read_word(root, PCI_EXP_LNKCTL2,
2137                                                           &tmp16);
2138                                 tmp16 &= ~(PCI_EXP_LNKCTL2_ENTER_COMP |
2139                                            PCI_EXP_LNKCTL2_TX_MARGIN);
2140                                 tmp16 |= (bridge_cfg2 &
2141                                           (PCI_EXP_LNKCTL2_ENTER_COMP |
2142                                            PCI_EXP_LNKCTL2_TX_MARGIN));
2143                                 pcie_capability_write_word(root,
2144                                                            PCI_EXP_LNKCTL2,
2145                                                            tmp16);
2146
2147                                 pcie_capability_read_word(adev->pdev,
2148                                                           PCI_EXP_LNKCTL2,
2149                                                           &tmp16);
2150                                 tmp16 &= ~(PCI_EXP_LNKCTL2_ENTER_COMP |
2151                                            PCI_EXP_LNKCTL2_TX_MARGIN);
2152                                 tmp16 |= (gpu_cfg2 &
2153                                           (PCI_EXP_LNKCTL2_ENTER_COMP |
2154                                            PCI_EXP_LNKCTL2_TX_MARGIN));
2155                                 pcie_capability_write_word(adev->pdev,
2156                                                            PCI_EXP_LNKCTL2,
2157                                                            tmp16);
2158
2159                                 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
2160                                 tmp &= ~LC_SET_QUIESCE;
2161                                 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
2162                         }
2163                 }
2164         }
2165
2166         speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
2167         speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
2168         WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
2169
2170         pcie_capability_read_word(adev->pdev, PCI_EXP_LNKCTL2, &tmp16);
2171         tmp16 &= ~PCI_EXP_LNKCTL2_TLS;
2172
2173         if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)
2174                 tmp16 |= PCI_EXP_LNKCTL2_TLS_8_0GT; /* gen3 */
2175         else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2)
2176                 tmp16 |= PCI_EXP_LNKCTL2_TLS_5_0GT; /* gen2 */
2177         else
2178                 tmp16 |= PCI_EXP_LNKCTL2_TLS_2_5GT; /* gen1 */
2179         pcie_capability_write_word(adev->pdev, PCI_EXP_LNKCTL2, tmp16);
2180
2181         speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
2182         speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
2183         WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
2184
2185         for (i = 0; i < adev->usec_timeout; i++) {
2186                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
2187                 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
2188                         break;
2189                 udelay(1);
2190         }
2191 }
2192
2193 static inline u32 si_pif_phy0_rreg(struct amdgpu_device *adev, u32 reg)
2194 {
2195         unsigned long flags;
2196         u32 r;
2197
2198         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
2199         WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
2200         r = RREG32(EVERGREEN_PIF_PHY0_DATA);
2201         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
2202         return r;
2203 }
2204
2205 static inline void si_pif_phy0_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
2206 {
2207         unsigned long flags;
2208
2209         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
2210         WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
2211         WREG32(EVERGREEN_PIF_PHY0_DATA, (v));
2212         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
2213 }
2214
2215 static inline u32 si_pif_phy1_rreg(struct amdgpu_device *adev, u32 reg)
2216 {
2217         unsigned long flags;
2218         u32 r;
2219
2220         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
2221         WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
2222         r = RREG32(EVERGREEN_PIF_PHY1_DATA);
2223         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
2224         return r;
2225 }
2226
2227 static inline void si_pif_phy1_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
2228 {
2229         unsigned long flags;
2230
2231         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
2232         WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
2233         WREG32(EVERGREEN_PIF_PHY1_DATA, (v));
2234         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
2235 }
2236 static void si_program_aspm(struct amdgpu_device *adev)
2237 {
2238         u32 data, orig;
2239         bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
2240         bool disable_clkreq = false;
2241
2242         if (amdgpu_aspm == 0)
2243                 return;
2244
2245         if (adev->flags & AMD_IS_APU)
2246                 return;
2247         orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
2248         data &= ~LC_XMIT_N_FTS_MASK;
2249         data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
2250         if (orig != data)
2251                 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
2252
2253         orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
2254         data |= LC_GO_TO_RECOVERY;
2255         if (orig != data)
2256                 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
2257
2258         orig = data = RREG32_PCIE(PCIE_P_CNTL);
2259         data |= P_IGNORE_EDB_ERR;
2260         if (orig != data)
2261                 WREG32_PCIE(PCIE_P_CNTL, data);
2262
2263         orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
2264         data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
2265         data |= LC_PMI_TO_L1_DIS;
2266         if (!disable_l0s)
2267                 data |= LC_L0S_INACTIVITY(7);
2268
2269         if (!disable_l1) {
2270                 data |= LC_L1_INACTIVITY(7);
2271                 data &= ~LC_PMI_TO_L1_DIS;
2272                 if (orig != data)
2273                         WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
2274
2275                 if (!disable_plloff_in_l1) {
2276                         bool clk_req_support;
2277
2278                         orig = data = si_pif_phy0_rreg(adev,PB0_PIF_PWRDOWN_0);
2279                         data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
2280                         data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
2281                         if (orig != data)
2282                                 si_pif_phy0_wreg(adev,PB0_PIF_PWRDOWN_0, data);
2283
2284                         orig = data = si_pif_phy0_rreg(adev,PB0_PIF_PWRDOWN_1);
2285                         data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
2286                         data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
2287                         if (orig != data)
2288                                 si_pif_phy0_wreg(adev,PB0_PIF_PWRDOWN_1, data);
2289
2290                         orig = data = si_pif_phy1_rreg(adev,PB1_PIF_PWRDOWN_0);
2291                         data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
2292                         data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
2293                         if (orig != data)
2294                                 si_pif_phy1_wreg(adev,PB1_PIF_PWRDOWN_0, data);
2295
2296                         orig = data = si_pif_phy1_rreg(adev,PB1_PIF_PWRDOWN_1);
2297                         data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
2298                         data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
2299                         if (orig != data)
2300                                 si_pif_phy1_wreg(adev,PB1_PIF_PWRDOWN_1, data);
2301
2302                         if ((adev->asic_type != CHIP_OLAND) && (adev->asic_type != CHIP_HAINAN)) {
2303                                 orig = data = si_pif_phy0_rreg(adev,PB0_PIF_PWRDOWN_0);
2304                                 data &= ~PLL_RAMP_UP_TIME_0_MASK;
2305                                 if (orig != data)
2306                                         si_pif_phy0_wreg(adev,PB0_PIF_PWRDOWN_0, data);
2307
2308                                 orig = data = si_pif_phy0_rreg(adev,PB0_PIF_PWRDOWN_1);
2309                                 data &= ~PLL_RAMP_UP_TIME_1_MASK;
2310                                 if (orig != data)
2311                                         si_pif_phy0_wreg(adev,PB0_PIF_PWRDOWN_1, data);
2312
2313                                 orig = data = si_pif_phy0_rreg(adev,PB0_PIF_PWRDOWN_2);
2314                                 data &= ~PLL_RAMP_UP_TIME_2_MASK;
2315                                 if (orig != data)
2316                                         si_pif_phy0_wreg(adev,PB0_PIF_PWRDOWN_2, data);
2317
2318                                 orig = data = si_pif_phy0_rreg(adev,PB0_PIF_PWRDOWN_3);
2319                                 data &= ~PLL_RAMP_UP_TIME_3_MASK;
2320                                 if (orig != data)
2321                                         si_pif_phy0_wreg(adev,PB0_PIF_PWRDOWN_3, data);
2322
2323                                 orig = data = si_pif_phy1_rreg(adev,PB1_PIF_PWRDOWN_0);
2324                                 data &= ~PLL_RAMP_UP_TIME_0_MASK;
2325                                 if (orig != data)
2326                                         si_pif_phy1_wreg(adev,PB1_PIF_PWRDOWN_0, data);
2327
2328                                 orig = data = si_pif_phy1_rreg(adev,PB1_PIF_PWRDOWN_1);
2329                                 data &= ~PLL_RAMP_UP_TIME_1_MASK;
2330                                 if (orig != data)
2331                                         si_pif_phy1_wreg(adev,PB1_PIF_PWRDOWN_1, data);
2332
2333                                 orig = data = si_pif_phy1_rreg(adev,PB1_PIF_PWRDOWN_2);
2334                                 data &= ~PLL_RAMP_UP_TIME_2_MASK;
2335                                 if (orig != data)
2336                                         si_pif_phy1_wreg(adev,PB1_PIF_PWRDOWN_2, data);
2337
2338                                 orig = data = si_pif_phy1_rreg(adev,PB1_PIF_PWRDOWN_3);
2339                                 data &= ~PLL_RAMP_UP_TIME_3_MASK;
2340                                 if (orig != data)
2341                                         si_pif_phy1_wreg(adev,PB1_PIF_PWRDOWN_3, data);
2342                         }
2343                         orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
2344                         data &= ~LC_DYN_LANES_PWR_STATE_MASK;
2345                         data |= LC_DYN_LANES_PWR_STATE(3);
2346                         if (orig != data)
2347                                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
2348
2349                         orig = data = si_pif_phy0_rreg(adev,PB0_PIF_CNTL);
2350                         data &= ~LS2_EXIT_TIME_MASK;
2351                         if ((adev->asic_type == CHIP_OLAND) || (adev->asic_type == CHIP_HAINAN))
2352                                 data |= LS2_EXIT_TIME(5);
2353                         if (orig != data)
2354                                 si_pif_phy0_wreg(adev,PB0_PIF_CNTL, data);
2355
2356                         orig = data = si_pif_phy1_rreg(adev,PB1_PIF_CNTL);
2357                         data &= ~LS2_EXIT_TIME_MASK;
2358                         if ((adev->asic_type == CHIP_OLAND) || (adev->asic_type == CHIP_HAINAN))
2359                                 data |= LS2_EXIT_TIME(5);
2360                         if (orig != data)
2361                                 si_pif_phy1_wreg(adev,PB1_PIF_CNTL, data);
2362
2363                         if (!disable_clkreq &&
2364                             !pci_is_root_bus(adev->pdev->bus)) {
2365                                 struct pci_dev *root = adev->pdev->bus->self;
2366                                 u32 lnkcap;
2367
2368                                 clk_req_support = false;
2369                                 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
2370                                 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
2371                                         clk_req_support = true;
2372                         } else {
2373                                 clk_req_support = false;
2374                         }
2375
2376                         if (clk_req_support) {
2377                                 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
2378                                 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
2379                                 if (orig != data)
2380                                         WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
2381
2382                                 orig = data = RREG32(THM_CLK_CNTL);
2383                                 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
2384                                 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
2385                                 if (orig != data)
2386                                         WREG32(THM_CLK_CNTL, data);
2387
2388                                 orig = data = RREG32(MISC_CLK_CNTL);
2389                                 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
2390                                 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
2391                                 if (orig != data)
2392                                         WREG32(MISC_CLK_CNTL, data);
2393
2394                                 orig = data = RREG32(CG_CLKPIN_CNTL);
2395                                 data &= ~BCLK_AS_XCLK;
2396                                 if (orig != data)
2397                                         WREG32(CG_CLKPIN_CNTL, data);
2398
2399                                 orig = data = RREG32(CG_CLKPIN_CNTL_2);
2400                                 data &= ~FORCE_BIF_REFCLK_EN;
2401                                 if (orig != data)
2402                                         WREG32(CG_CLKPIN_CNTL_2, data);
2403
2404                                 orig = data = RREG32(MPLL_BYPASSCLK_SEL);
2405                                 data &= ~MPLL_CLKOUT_SEL_MASK;
2406                                 data |= MPLL_CLKOUT_SEL(4);
2407                                 if (orig != data)
2408                                         WREG32(MPLL_BYPASSCLK_SEL, data);
2409
2410                                 orig = data = RREG32(SPLL_CNTL_MODE);
2411                                 data &= ~SPLL_REFCLK_SEL_MASK;
2412                                 if (orig != data)
2413                                         WREG32(SPLL_CNTL_MODE, data);
2414                         }
2415                 }
2416         } else {
2417                 if (orig != data)
2418                         WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
2419         }
2420
2421         orig = data = RREG32_PCIE(PCIE_CNTL2);
2422         data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
2423         if (orig != data)
2424                 WREG32_PCIE(PCIE_CNTL2, data);
2425
2426         if (!disable_l0s) {
2427                 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
2428                 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
2429                         data = RREG32_PCIE(PCIE_LC_STATUS1);
2430                         if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
2431                                 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
2432                                 data &= ~LC_L0S_INACTIVITY_MASK;
2433                                 if (orig != data)
2434                                         WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
2435                         }
2436                 }
2437         }
2438 }
2439
2440 static void si_fix_pci_max_read_req_size(struct amdgpu_device *adev)
2441 {
2442         int readrq;
2443         u16 v;
2444
2445         readrq = pcie_get_readrq(adev->pdev);
2446         v = ffs(readrq) - 8;
2447         if ((v == 0) || (v == 6) || (v == 7))
2448                 pcie_set_readrq(adev->pdev, 512);
2449 }
2450
2451 static int si_common_hw_init(void *handle)
2452 {
2453         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2454
2455         si_fix_pci_max_read_req_size(adev);
2456         si_init_golden_registers(adev);
2457         si_pcie_gen3_enable(adev);
2458         si_program_aspm(adev);
2459
2460         return 0;
2461 }
2462
2463 static int si_common_hw_fini(void *handle)
2464 {
2465         return 0;
2466 }
2467
2468 static int si_common_suspend(void *handle)
2469 {
2470         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2471
2472         return si_common_hw_fini(adev);
2473 }
2474
2475 static int si_common_resume(void *handle)
2476 {
2477         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2478
2479         return si_common_hw_init(adev);
2480 }
2481
2482 static bool si_common_is_idle(void *handle)
2483 {
2484         return true;
2485 }
2486
2487 static int si_common_wait_for_idle(void *handle)
2488 {
2489         return 0;
2490 }
2491
2492 static int si_common_soft_reset(void *handle)
2493 {
2494         return 0;
2495 }
2496
2497 static int si_common_set_clockgating_state(void *handle,
2498                                             enum amd_clockgating_state state)
2499 {
2500         return 0;
2501 }
2502
2503 static int si_common_set_powergating_state(void *handle,
2504                                             enum amd_powergating_state state)
2505 {
2506         return 0;
2507 }
2508
2509 static const struct amd_ip_funcs si_common_ip_funcs = {
2510         .name = "si_common",
2511         .early_init = si_common_early_init,
2512         .late_init = NULL,
2513         .sw_init = si_common_sw_init,
2514         .sw_fini = si_common_sw_fini,
2515         .hw_init = si_common_hw_init,
2516         .hw_fini = si_common_hw_fini,
2517         .suspend = si_common_suspend,
2518         .resume = si_common_resume,
2519         .is_idle = si_common_is_idle,
2520         .wait_for_idle = si_common_wait_for_idle,
2521         .soft_reset = si_common_soft_reset,
2522         .set_clockgating_state = si_common_set_clockgating_state,
2523         .set_powergating_state = si_common_set_powergating_state,
2524 };
2525
2526 static const struct amdgpu_ip_block_version si_common_ip_block =
2527 {
2528         .type = AMD_IP_BLOCK_TYPE_COMMON,
2529         .major = 1,
2530         .minor = 0,
2531         .rev = 0,
2532         .funcs = &si_common_ip_funcs,
2533 };
2534
2535 int si_set_ip_blocks(struct amdgpu_device *adev)
2536 {
2537         switch (adev->asic_type) {
2538         case CHIP_VERDE:
2539         case CHIP_TAHITI:
2540         case CHIP_PITCAIRN:
2541                 amdgpu_device_ip_block_add(adev, &si_common_ip_block);
2542                 amdgpu_device_ip_block_add(adev, &gmc_v6_0_ip_block);
2543                 amdgpu_device_ip_block_add(adev, &si_ih_ip_block);
2544                 amdgpu_device_ip_block_add(adev, &gfx_v6_0_ip_block);
2545                 amdgpu_device_ip_block_add(adev, &si_dma_ip_block);
2546                 amdgpu_device_ip_block_add(adev, &si_smu_ip_block);
2547                 if (adev->enable_virtual_display)
2548                         amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block);
2549                 else
2550                         amdgpu_device_ip_block_add(adev, &dce_v6_0_ip_block);
2551                 amdgpu_device_ip_block_add(adev, &uvd_v3_1_ip_block);
2552                 /* amdgpu_device_ip_block_add(adev, &vce_v1_0_ip_block); */
2553                 break;
2554         case CHIP_OLAND:
2555                 amdgpu_device_ip_block_add(adev, &si_common_ip_block);
2556                 amdgpu_device_ip_block_add(adev, &gmc_v6_0_ip_block);
2557                 amdgpu_device_ip_block_add(adev, &si_ih_ip_block);
2558                 amdgpu_device_ip_block_add(adev, &gfx_v6_0_ip_block);
2559                 amdgpu_device_ip_block_add(adev, &si_dma_ip_block);
2560                 amdgpu_device_ip_block_add(adev, &si_smu_ip_block);
2561                 if (adev->enable_virtual_display)
2562                         amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block);
2563                 else
2564                         amdgpu_device_ip_block_add(adev, &dce_v6_4_ip_block);
2565                 amdgpu_device_ip_block_add(adev, &uvd_v3_1_ip_block);
2566                 /* amdgpu_device_ip_block_add(adev, &vce_v1_0_ip_block); */
2567                 break;
2568         case CHIP_HAINAN:
2569                 amdgpu_device_ip_block_add(adev, &si_common_ip_block);
2570                 amdgpu_device_ip_block_add(adev, &gmc_v6_0_ip_block);
2571                 amdgpu_device_ip_block_add(adev, &si_ih_ip_block);
2572                 amdgpu_device_ip_block_add(adev, &gfx_v6_0_ip_block);
2573                 amdgpu_device_ip_block_add(adev, &si_dma_ip_block);
2574                 amdgpu_device_ip_block_add(adev, &si_smu_ip_block);
2575                 if (adev->enable_virtual_display)
2576                         amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block);
2577                 break;
2578         default:
2579                 BUG();
2580         }
2581         return 0;
2582 }
2583