drivers/gpu/drm/amd/amdgpu/cik.c

   1 /*
   2  * Copyright 2012 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Alex Deucher
  23  */
  24 #include <linux/firmware.h>
  25 #include <linux/slab.h>
  26 #include <linux/module.h>
  27 #include <linux/pci.h>
  28
  29 #include <drm/amdgpu_drm.h>
  30
  31 #include "amdgpu.h"
  32 #include "amdgpu_atombios.h"
  33 #include "amdgpu_ih.h"
  34 #include "amdgpu_uvd.h"
  35 #include "amdgpu_vce.h"
  36 #include "cikd.h"
  37 #include "atom.h"
  38 #include "amd_pcie.h"
  39
  40 #include "cik.h"
  41 #include "gmc_v7_0.h"
  42 #include "cik_ih.h"
  43 #include "dce_v8_0.h"
  44 #include "gfx_v7_0.h"
  45 #include "cik_sdma.h"
  46 #include "uvd_v4_2.h"
  47 #include "vce_v2_0.h"
  48 #include "cik_dpm.h"
  49
  50 #include "uvd/uvd_4_2_d.h"
  51
  52 #include "smu/smu_7_0_1_d.h"
  53 #include "smu/smu_7_0_1_sh_mask.h"
  54
  55 #include "dce/dce_8_0_d.h"
  56 #include "dce/dce_8_0_sh_mask.h"
  57
  58 #include "bif/bif_4_1_d.h"
  59 #include "bif/bif_4_1_sh_mask.h"
  60
  61 #include "gca/gfx_7_2_d.h"
  62 #include "gca/gfx_7_2_enum.h"
  63 #include "gca/gfx_7_2_sh_mask.h"
  64
  65 #include "gmc/gmc_7_1_d.h"
  66 #include "gmc/gmc_7_1_sh_mask.h"
  67
  68 #include "oss/oss_2_0_d.h"
  69 #include "oss/oss_2_0_sh_mask.h"
  70
  71 #include "amdgpu_dm.h"
  72 #include "amdgpu_amdkfd.h"
  73 #include "amdgpu_vkms.h"
  74
  75 static const struct amdgpu_video_codec_info cik_video_codecs_encode_array[] =
  76 {
  77         {
  78                 .codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4_AVC,
  79                 .max_width = 2048,
  80                 .max_height = 1152,
  81                 .max_pixels_per_frame = 2048 * 1152,
  82                 .max_level = 0,
  83         },
  84 };
  85
  86 static const struct amdgpu_video_codecs cik_video_codecs_encode =
  87 {
  88         .codec_count = ARRAY_SIZE(cik_video_codecs_encode_array),
  89         .codec_array = cik_video_codecs_encode_array,
  90 };
  91
  92 static const struct amdgpu_video_codec_info cik_video_codecs_decode_array[] =
  93 {
  94         {
  95                 .codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG2,
  96                 .max_width = 2048,
  97                 .max_height = 1152,
  98                 .max_pixels_per_frame = 2048 * 1152,
  99                 .max_level = 3,
 100         },
 101         {
 102                 .codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4,
 103                 .max_width = 2048,
 104                 .max_height = 1152,
 105                 .max_pixels_per_frame = 2048 * 1152,
 106                 .max_level = 5,
 107         },
 108         {
 109                 .codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4_AVC,
 110                 .max_width = 2048,
 111                 .max_height = 1152,
 112                 .max_pixels_per_frame = 2048 * 1152,
 113                 .max_level = 41,
 114         },
 115         {
 116                 .codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_VC1,
 117                 .max_width = 2048,
 118                 .max_height = 1152,
 119                 .max_pixels_per_frame = 2048 * 1152,
 120                 .max_level = 4,
 121         },
 122 };
 123
 124 static const struct amdgpu_video_codecs cik_video_codecs_decode =
 125 {
 126         .codec_count = ARRAY_SIZE(cik_video_codecs_decode_array),
 127         .codec_array = cik_video_codecs_decode_array,
 128 };
 129
 130 static int cik_query_video_codecs(struct amdgpu_device *adev, bool encode,
 131                                   const struct amdgpu_video_codecs **codecs)
 132 {
 133         switch (adev->asic_type) {
 134         case CHIP_BONAIRE:
 135         case CHIP_HAWAII:
 136         case CHIP_KAVERI:
 137         case CHIP_KABINI:
 138         case CHIP_MULLINS:
 139                 if (encode)
 140                         *codecs = &cik_video_codecs_encode;
 141                 else
 142                         *codecs = &cik_video_codecs_decode;
 143                 return 0;
 144         default:
 145                 return -EINVAL;
 146         }
 147 }
 148
 149 /*
 150  * Indirect registers accessor
 151  */
 152 static u32 cik_pcie_rreg(struct amdgpu_device *adev, u32 reg)
 153 {
 154         unsigned long flags;
 155         u32 r;
 156
 157         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 158         WREG32(mmPCIE_INDEX, reg);
 159         (void)RREG32(mmPCIE_INDEX);
 160         r = RREG32(mmPCIE_DATA);
 161         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 162         return r;
 163 }
 164
 165 static void cik_pcie_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 166 {
 167         unsigned long flags;
 168
 169         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 170         WREG32(mmPCIE_INDEX, reg);
 171         (void)RREG32(mmPCIE_INDEX);
 172         WREG32(mmPCIE_DATA, v);
 173         (void)RREG32(mmPCIE_DATA);
 174         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 175 }
 176
 177 static u32 cik_smc_rreg(struct amdgpu_device *adev, u32 reg)
 178 {
 179         unsigned long flags;
 180         u32 r;
 181
 182         spin_lock_irqsave(&adev->smc_idx_lock, flags);
 183         WREG32(mmSMC_IND_INDEX_0, (reg));
 184         r = RREG32(mmSMC_IND_DATA_0);
 185         spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
 186         return r;
 187 }
 188
 189 static void cik_smc_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 190 {
 191         unsigned long flags;
 192
 193         spin_lock_irqsave(&adev->smc_idx_lock, flags);
 194         WREG32(mmSMC_IND_INDEX_0, (reg));
 195         WREG32(mmSMC_IND_DATA_0, (v));
 196         spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
 197 }
 198
 199 static u32 cik_uvd_ctx_rreg(struct amdgpu_device *adev, u32 reg)
 200 {
 201         unsigned long flags;
 202         u32 r;
 203
 204         spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
 205         WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
 206         r = RREG32(mmUVD_CTX_DATA);
 207         spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
 208         return r;
 209 }
 210
 211 static void cik_uvd_ctx_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 212 {
 213         unsigned long flags;
 214
 215         spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
 216         WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
 217         WREG32(mmUVD_CTX_DATA, (v));
 218         spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
 219 }
 220
 221 static u32 cik_didt_rreg(struct amdgpu_device *adev, u32 reg)
 222 {
 223         unsigned long flags;
 224         u32 r;
 225
 226         spin_lock_irqsave(&adev->didt_idx_lock, flags);
 227         WREG32(mmDIDT_IND_INDEX, (reg));
 228         r = RREG32(mmDIDT_IND_DATA);
 229         spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
 230         return r;
 231 }
 232
 233 static void cik_didt_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 234 {
 235         unsigned long flags;
 236
 237         spin_lock_irqsave(&adev->didt_idx_lock, flags);
 238         WREG32(mmDIDT_IND_INDEX, (reg));
 239         WREG32(mmDIDT_IND_DATA, (v));
 240         spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
 241 }
 242
 243 static const u32 bonaire_golden_spm_registers[] =
 244 {
 245         0xc200, 0xe0ffffff, 0xe0000000
 246 };
 247
 248 static const u32 bonaire_golden_common_registers[] =
 249 {
 250         0x31dc, 0xffffffff, 0x00000800,
 251         0x31dd, 0xffffffff, 0x00000800,
 252         0x31e6, 0xffffffff, 0x00007fbf,
 253         0x31e7, 0xffffffff, 0x00007faf
 254 };
 255
 256 static const u32 bonaire_golden_registers[] =
 257 {
 258         0xcd5, 0x00000333, 0x00000333,
 259         0xcd4, 0x000c0fc0, 0x00040200,
 260         0x2684, 0x00010000, 0x00058208,
 261         0xf000, 0xffff1fff, 0x00140000,
 262         0xf080, 0xfdfc0fff, 0x00000100,
 263         0xf08d, 0x40000000, 0x40000200,
 264         0x260c, 0xffffffff, 0x00000000,
 265         0x260d, 0xf00fffff, 0x00000400,
 266         0x260e, 0x0002021c, 0x00020200,
 267         0x31e, 0x00000080, 0x00000000,
 268         0x16ec, 0x000000f0, 0x00000070,
 269         0x16f0, 0xf0311fff, 0x80300000,
 270         0x263e, 0x73773777, 0x12010001,
 271         0xd43, 0x00810000, 0x408af000,
 272         0x1c0c, 0x31000111, 0x00000011,
 273         0xbd2, 0x73773777, 0x12010001,
 274         0x883, 0x00007fb6, 0x0021a1b1,
 275         0x884, 0x00007fb6, 0x002021b1,
 276         0x860, 0x00007fb6, 0x00002191,
 277         0x886, 0x00007fb6, 0x002121b1,
 278         0x887, 0x00007fb6, 0x002021b1,
 279         0x877, 0x00007fb6, 0x00002191,
 280         0x878, 0x00007fb6, 0x00002191,
 281         0xd8a, 0x0000003f, 0x0000000a,
 282         0xd8b, 0x0000003f, 0x0000000a,
 283         0xab9, 0x00073ffe, 0x000022a2,
 284         0x903, 0x000007ff, 0x00000000,
 285         0x2285, 0xf000003f, 0x00000007,
 286         0x22fc, 0x00002001, 0x00000001,
 287         0x22c9, 0xffffffff, 0x00ffffff,
 288         0xc281, 0x0000ff0f, 0x00000000,
 289         0xa293, 0x07ffffff, 0x06000000,
 290         0x136, 0x00000fff, 0x00000100,
 291         0xf9e, 0x00000001, 0x00000002,
 292         0x2440, 0x03000000, 0x0362c688,
 293         0x2300, 0x000000ff, 0x00000001,
 294         0x390, 0x00001fff, 0x00001fff,
 295         0x2418, 0x0000007f, 0x00000020,
 296         0x2542, 0x00010000, 0x00010000,
 297         0x2b05, 0x000003ff, 0x000000f3,
 298         0x2b03, 0xffffffff, 0x00001032
 299 };
 300
 301 static const u32 bonaire_mgcg_cgcg_init[] =
 302 {
 303         0x3108, 0xffffffff, 0xfffffffc,
 304         0xc200, 0xffffffff, 0xe0000000,
 305         0xf0a8, 0xffffffff, 0x00000100,
 306         0xf082, 0xffffffff, 0x00000100,
 307         0xf0b0, 0xffffffff, 0xc0000100,
 308         0xf0b2, 0xffffffff, 0xc0000100,
 309         0xf0b1, 0xffffffff, 0xc0000100,
 310         0x1579, 0xffffffff, 0x00600100,
 311         0xf0a0, 0xffffffff, 0x00000100,
 312         0xf085, 0xffffffff, 0x06000100,
 313         0xf088, 0xffffffff, 0x00000100,
 314         0xf086, 0xffffffff, 0x06000100,
 315         0xf081, 0xffffffff, 0x00000100,
 316         0xf0b8, 0xffffffff, 0x00000100,
 317         0xf089, 0xffffffff, 0x00000100,
 318         0xf080, 0xffffffff, 0x00000100,
 319         0xf08c, 0xffffffff, 0x00000100,
 320         0xf08d, 0xffffffff, 0x00000100,
 321         0xf094, 0xffffffff, 0x00000100,
 322         0xf095, 0xffffffff, 0x00000100,
 323         0xf096, 0xffffffff, 0x00000100,
 324         0xf097, 0xffffffff, 0x00000100,
 325         0xf098, 0xffffffff, 0x00000100,
 326         0xf09f, 0xffffffff, 0x00000100,
 327         0xf09e, 0xffffffff, 0x00000100,
 328         0xf084, 0xffffffff, 0x06000100,
 329         0xf0a4, 0xffffffff, 0x00000100,
 330         0xf09d, 0xffffffff, 0x00000100,
 331         0xf0ad, 0xffffffff, 0x00000100,
 332         0xf0ac, 0xffffffff, 0x00000100,
 333         0xf09c, 0xffffffff, 0x00000100,
 334         0xc200, 0xffffffff, 0xe0000000,
 335         0xf008, 0xffffffff, 0x00010000,
 336         0xf009, 0xffffffff, 0x00030002,
 337         0xf00a, 0xffffffff, 0x00040007,
 338         0xf00b, 0xffffffff, 0x00060005,
 339         0xf00c, 0xffffffff, 0x00090008,
 340         0xf00d, 0xffffffff, 0x00010000,
 341         0xf00e, 0xffffffff, 0x00030002,
 342         0xf00f, 0xffffffff, 0x00040007,
 343         0xf010, 0xffffffff, 0x00060005,
 344         0xf011, 0xffffffff, 0x00090008,
 345         0xf012, 0xffffffff, 0x00010000,
 346         0xf013, 0xffffffff, 0x00030002,
 347         0xf014, 0xffffffff, 0x00040007,
 348         0xf015, 0xffffffff, 0x00060005,
 349         0xf016, 0xffffffff, 0x00090008,
 350         0xf017, 0xffffffff, 0x00010000,
 351         0xf018, 0xffffffff, 0x00030002,
 352         0xf019, 0xffffffff, 0x00040007,
 353         0xf01a, 0xffffffff, 0x00060005,
 354         0xf01b, 0xffffffff, 0x00090008,
 355         0xf01c, 0xffffffff, 0x00010000,
 356         0xf01d, 0xffffffff, 0x00030002,
 357         0xf01e, 0xffffffff, 0x00040007,
 358         0xf01f, 0xffffffff, 0x00060005,
 359         0xf020, 0xffffffff, 0x00090008,
 360         0xf021, 0xffffffff, 0x00010000,
 361         0xf022, 0xffffffff, 0x00030002,
 362         0xf023, 0xffffffff, 0x00040007,
 363         0xf024, 0xffffffff, 0x00060005,
 364         0xf025, 0xffffffff, 0x00090008,
 365         0xf026, 0xffffffff, 0x00010000,
 366         0xf027, 0xffffffff, 0x00030002,
 367         0xf028, 0xffffffff, 0x00040007,
 368         0xf029, 0xffffffff, 0x00060005,
 369         0xf02a, 0xffffffff, 0x00090008,
 370         0xf000, 0xffffffff, 0x96e00200,
 371         0x21c2, 0xffffffff, 0x00900100,
 372         0x3109, 0xffffffff, 0x0020003f,
 373         0xe, 0xffffffff, 0x0140001c,
 374         0xf, 0x000f0000, 0x000f0000,
 375         0x88, 0xffffffff, 0xc060000c,
 376         0x89, 0xc0000fff, 0x00000100,
 377         0x3e4, 0xffffffff, 0x00000100,
 378         0x3e6, 0x00000101, 0x00000000,
 379         0x82a, 0xffffffff, 0x00000104,
 380         0x1579, 0xff000fff, 0x00000100,
 381         0xc33, 0xc0000fff, 0x00000104,
 382         0x3079, 0x00000001, 0x00000001,
 383         0x3403, 0xff000ff0, 0x00000100,
 384         0x3603, 0xff000ff0, 0x00000100
 385 };
 386
 387 static const u32 spectre_golden_spm_registers[] =
 388 {
 389         0xc200, 0xe0ffffff, 0xe0000000
 390 };
 391
 392 static const u32 spectre_golden_common_registers[] =
 393 {
 394         0x31dc, 0xffffffff, 0x00000800,
 395         0x31dd, 0xffffffff, 0x00000800,
 396         0x31e6, 0xffffffff, 0x00007fbf,
 397         0x31e7, 0xffffffff, 0x00007faf
 398 };
 399
 400 static const u32 spectre_golden_registers[] =
 401 {
 402         0xf000, 0xffff1fff, 0x96940200,
 403         0xf003, 0xffff0001, 0xff000000,
 404         0xf080, 0xfffc0fff, 0x00000100,
 405         0x1bb6, 0x00010101, 0x00010000,
 406         0x260d, 0xf00fffff, 0x00000400,
 407         0x260e, 0xfffffffc, 0x00020200,
 408         0x16ec, 0x000000f0, 0x00000070,
 409         0x16f0, 0xf0311fff, 0x80300000,
 410         0x263e, 0x73773777, 0x12010001,
 411         0x26df, 0x00ff0000, 0x00fc0000,
 412         0xbd2, 0x73773777, 0x12010001,
 413         0x2285, 0xf000003f, 0x00000007,
 414         0x22c9, 0xffffffff, 0x00ffffff,
 415         0xa0d4, 0x3f3f3fff, 0x00000082,
 416         0xa0d5, 0x0000003f, 0x00000000,
 417         0xf9e, 0x00000001, 0x00000002,
 418         0x244f, 0xffff03df, 0x00000004,
 419         0x31da, 0x00000008, 0x00000008,
 420         0x2300, 0x000008ff, 0x00000800,
 421         0x2542, 0x00010000, 0x00010000,
 422         0x2b03, 0xffffffff, 0x54763210,
 423         0x853e, 0x01ff01ff, 0x00000002,
 424         0x8526, 0x007ff800, 0x00200000,
 425         0x8057, 0xffffffff, 0x00000f40,
 426         0xc24d, 0xffffffff, 0x00000001
 427 };
 428
 429 static const u32 spectre_mgcg_cgcg_init[] =
 430 {
 431         0x3108, 0xffffffff, 0xfffffffc,
 432         0xc200, 0xffffffff, 0xe0000000,
 433         0xf0a8, 0xffffffff, 0x00000100,
 434         0xf082, 0xffffffff, 0x00000100,
 435         0xf0b0, 0xffffffff, 0x00000100,
 436         0xf0b2, 0xffffffff, 0x00000100,
 437         0xf0b1, 0xffffffff, 0x00000100,
 438         0x1579, 0xffffffff, 0x00600100,
 439         0xf0a0, 0xffffffff, 0x00000100,
 440         0xf085, 0xffffffff, 0x06000100,
 441         0xf088, 0xffffffff, 0x00000100,
 442         0xf086, 0xffffffff, 0x06000100,
 443         0xf081, 0xffffffff, 0x00000100,
 444         0xf0b8, 0xffffffff, 0x00000100,
 445         0xf089, 0xffffffff, 0x00000100,
 446         0xf080, 0xffffffff, 0x00000100,
 447         0xf08c, 0xffffffff, 0x00000100,
 448         0xf08d, 0xffffffff, 0x00000100,
 449         0xf094, 0xffffffff, 0x00000100,
 450         0xf095, 0xffffffff, 0x00000100,
 451         0xf096, 0xffffffff, 0x00000100,
 452         0xf097, 0xffffffff, 0x00000100,
 453         0xf098, 0xffffffff, 0x00000100,
 454         0xf09f, 0xffffffff, 0x00000100,
 455         0xf09e, 0xffffffff, 0x00000100,
 456         0xf084, 0xffffffff, 0x06000100,
 457         0xf0a4, 0xffffffff, 0x00000100,
 458         0xf09d, 0xffffffff, 0x00000100,
 459         0xf0ad, 0xffffffff, 0x00000100,
 460         0xf0ac, 0xffffffff, 0x00000100,
 461         0xf09c, 0xffffffff, 0x00000100,
 462         0xc200, 0xffffffff, 0xe0000000,
 463         0xf008, 0xffffffff, 0x00010000,
 464         0xf009, 0xffffffff, 0x00030002,
 465         0xf00a, 0xffffffff, 0x00040007,
 466         0xf00b, 0xffffffff, 0x00060005,
 467         0xf00c, 0xffffffff, 0x00090008,
 468         0xf00d, 0xffffffff, 0x00010000,
 469         0xf00e, 0xffffffff, 0x00030002,
 470         0xf00f, 0xffffffff, 0x00040007,
 471         0xf010, 0xffffffff, 0x00060005,
 472         0xf011, 0xffffffff, 0x00090008,
 473         0xf012, 0xffffffff, 0x00010000,
 474         0xf013, 0xffffffff, 0x00030002,
 475         0xf014, 0xffffffff, 0x00040007,
 476         0xf015, 0xffffffff, 0x00060005,
 477         0xf016, 0xffffffff, 0x00090008,
 478         0xf017, 0xffffffff, 0x00010000,
 479         0xf018, 0xffffffff, 0x00030002,
 480         0xf019, 0xffffffff, 0x00040007,
 481         0xf01a, 0xffffffff, 0x00060005,
 482         0xf01b, 0xffffffff, 0x00090008,
 483         0xf01c, 0xffffffff, 0x00010000,
 484         0xf01d, 0xffffffff, 0x00030002,
 485         0xf01e, 0xffffffff, 0x00040007,
 486         0xf01f, 0xffffffff, 0x00060005,
 487         0xf020, 0xffffffff, 0x00090008,
 488         0xf021, 0xffffffff, 0x00010000,
 489         0xf022, 0xffffffff, 0x00030002,
 490         0xf023, 0xffffffff, 0x00040007,
 491         0xf024, 0xffffffff, 0x00060005,
 492         0xf025, 0xffffffff, 0x00090008,
 493         0xf026, 0xffffffff, 0x00010000,
 494         0xf027, 0xffffffff, 0x00030002,
 495         0xf028, 0xffffffff, 0x00040007,
 496         0xf029, 0xffffffff, 0x00060005,
 497         0xf02a, 0xffffffff, 0x00090008,
 498         0xf02b, 0xffffffff, 0x00010000,
 499         0xf02c, 0xffffffff, 0x00030002,
 500         0xf02d, 0xffffffff, 0x00040007,
 501         0xf02e, 0xffffffff, 0x00060005,
 502         0xf02f, 0xffffffff, 0x00090008,
 503         0xf000, 0xffffffff, 0x96e00200,
 504         0x21c2, 0xffffffff, 0x00900100,
 505         0x3109, 0xffffffff, 0x0020003f,
 506         0xe, 0xffffffff, 0x0140001c,
 507         0xf, 0x000f0000, 0x000f0000,
 508         0x88, 0xffffffff, 0xc060000c,
 509         0x89, 0xc0000fff, 0x00000100,
 510         0x3e4, 0xffffffff, 0x00000100,
 511         0x3e6, 0x00000101, 0x00000000,
 512         0x82a, 0xffffffff, 0x00000104,
 513         0x1579, 0xff000fff, 0x00000100,
 514         0xc33, 0xc0000fff, 0x00000104,
 515         0x3079, 0x00000001, 0x00000001,
 516         0x3403, 0xff000ff0, 0x00000100,
 517         0x3603, 0xff000ff0, 0x00000100
 518 };
 519
 520 static const u32 kalindi_golden_spm_registers[] =
 521 {
 522         0xc200, 0xe0ffffff, 0xe0000000
 523 };
 524
 525 static const u32 kalindi_golden_common_registers[] =
 526 {
 527         0x31dc, 0xffffffff, 0x00000800,
 528         0x31dd, 0xffffffff, 0x00000800,
 529         0x31e6, 0xffffffff, 0x00007fbf,
 530         0x31e7, 0xffffffff, 0x00007faf
 531 };
 532
 533 static const u32 kalindi_golden_registers[] =
 534 {
 535         0xf000, 0xffffdfff, 0x6e944040,
 536         0x1579, 0xff607fff, 0xfc000100,
 537         0xf088, 0xff000fff, 0x00000100,
 538         0xf089, 0xff000fff, 0x00000100,
 539         0xf080, 0xfffc0fff, 0x00000100,
 540         0x1bb6, 0x00010101, 0x00010000,
 541         0x260c, 0xffffffff, 0x00000000,
 542         0x260d, 0xf00fffff, 0x00000400,
 543         0x16ec, 0x000000f0, 0x00000070,
 544         0x16f0, 0xf0311fff, 0x80300000,
 545         0x263e, 0x73773777, 0x12010001,
 546         0x263f, 0xffffffff, 0x00000010,
 547         0x26df, 0x00ff0000, 0x00fc0000,
 548         0x200c, 0x00001f0f, 0x0000100a,
 549         0xbd2, 0x73773777, 0x12010001,
 550         0x902, 0x000fffff, 0x000c007f,
 551         0x2285, 0xf000003f, 0x00000007,
 552         0x22c9, 0x3fff3fff, 0x00ffcfff,
 553         0xc281, 0x0000ff0f, 0x00000000,
 554         0xa293, 0x07ffffff, 0x06000000,
 555         0x136, 0x00000fff, 0x00000100,
 556         0xf9e, 0x00000001, 0x00000002,
 557         0x31da, 0x00000008, 0x00000008,
 558         0x2300, 0x000000ff, 0x00000003,
 559         0x853e, 0x01ff01ff, 0x00000002,
 560         0x8526, 0x007ff800, 0x00200000,
 561         0x8057, 0xffffffff, 0x00000f40,
 562         0x2231, 0x001f3ae3, 0x00000082,
 563         0x2235, 0x0000001f, 0x00000010,
 564         0xc24d, 0xffffffff, 0x00000000
 565 };
 566
 567 static const u32 kalindi_mgcg_cgcg_init[] =
 568 {
 569         0x3108, 0xffffffff, 0xfffffffc,
 570         0xc200, 0xffffffff, 0xe0000000,
 571         0xf0a8, 0xffffffff, 0x00000100,
 572         0xf082, 0xffffffff, 0x00000100,
 573         0xf0b0, 0xffffffff, 0x00000100,
 574         0xf0b2, 0xffffffff, 0x00000100,
 575         0xf0b1, 0xffffffff, 0x00000100,
 576         0x1579, 0xffffffff, 0x00600100,
 577         0xf0a0, 0xffffffff, 0x00000100,
 578         0xf085, 0xffffffff, 0x06000100,
 579         0xf088, 0xffffffff, 0x00000100,
 580         0xf086, 0xffffffff, 0x06000100,
 581         0xf081, 0xffffffff, 0x00000100,
 582         0xf0b8, 0xffffffff, 0x00000100,
 583         0xf089, 0xffffffff, 0x00000100,
 584         0xf080, 0xffffffff, 0x00000100,
 585         0xf08c, 0xffffffff, 0x00000100,
 586         0xf08d, 0xffffffff, 0x00000100,
 587         0xf094, 0xffffffff, 0x00000100,
 588         0xf095, 0xffffffff, 0x00000100,
 589         0xf096, 0xffffffff, 0x00000100,
 590         0xf097, 0xffffffff, 0x00000100,
 591         0xf098, 0xffffffff, 0x00000100,
 592         0xf09f, 0xffffffff, 0x00000100,
 593         0xf09e, 0xffffffff, 0x00000100,
 594         0xf084, 0xffffffff, 0x06000100,
 595         0xf0a4, 0xffffffff, 0x00000100,
 596         0xf09d, 0xffffffff, 0x00000100,
 597         0xf0ad, 0xffffffff, 0x00000100,
 598         0xf0ac, 0xffffffff, 0x00000100,
 599         0xf09c, 0xffffffff, 0x00000100,
 600         0xc200, 0xffffffff, 0xe0000000,
 601         0xf008, 0xffffffff, 0x00010000,
 602         0xf009, 0xffffffff, 0x00030002,
 603         0xf00a, 0xffffffff, 0x00040007,
 604         0xf00b, 0xffffffff, 0x00060005,
 605         0xf00c, 0xffffffff, 0x00090008,
 606         0xf00d, 0xffffffff, 0x00010000,
 607         0xf00e, 0xffffffff, 0x00030002,
 608         0xf00f, 0xffffffff, 0x00040007,
 609         0xf010, 0xffffffff, 0x00060005,
 610         0xf011, 0xffffffff, 0x00090008,
 611         0xf000, 0xffffffff, 0x96e00200,
 612         0x21c2, 0xffffffff, 0x00900100,
 613         0x3109, 0xffffffff, 0x0020003f,
 614         0xe, 0xffffffff, 0x0140001c,
 615         0xf, 0x000f0000, 0x000f0000,
 616         0x88, 0xffffffff, 0xc060000c,
 617         0x89, 0xc0000fff, 0x00000100,
 618         0x82a, 0xffffffff, 0x00000104,
 619         0x1579, 0xff000fff, 0x00000100,
 620         0xc33, 0xc0000fff, 0x00000104,
 621         0x3079, 0x00000001, 0x00000001,
 622         0x3403, 0xff000ff0, 0x00000100,
 623         0x3603, 0xff000ff0, 0x00000100
 624 };
 625
 626 static const u32 hawaii_golden_spm_registers[] =
 627 {
 628         0xc200, 0xe0ffffff, 0xe0000000
 629 };
 630
 631 static const u32 hawaii_golden_common_registers[] =
 632 {
 633         0xc200, 0xffffffff, 0xe0000000,
 634         0xa0d4, 0xffffffff, 0x3a00161a,
 635         0xa0d5, 0xffffffff, 0x0000002e,
 636         0x2684, 0xffffffff, 0x00018208,
 637         0x263e, 0xffffffff, 0x12011003
 638 };
 639
 640 static const u32 hawaii_golden_registers[] =
 641 {
 642         0xcd5, 0x00000333, 0x00000333,
 643         0x2684, 0x00010000, 0x00058208,
 644         0x260c, 0xffffffff, 0x00000000,
 645         0x260d, 0xf00fffff, 0x00000400,
 646         0x260e, 0x0002021c, 0x00020200,
 647         0x31e, 0x00000080, 0x00000000,
 648         0x16ec, 0x000000f0, 0x00000070,
 649         0x16f0, 0xf0311fff, 0x80300000,
 650         0xd43, 0x00810000, 0x408af000,
 651         0x1c0c, 0x31000111, 0x00000011,
 652         0xbd2, 0x73773777, 0x12010001,
 653         0x848, 0x0000007f, 0x0000001b,
 654         0x877, 0x00007fb6, 0x00002191,
 655         0xd8a, 0x0000003f, 0x0000000a,
 656         0xd8b, 0x0000003f, 0x0000000a,
 657         0xab9, 0x00073ffe, 0x000022a2,
 658         0x903, 0x000007ff, 0x00000000,
 659         0x22fc, 0x00002001, 0x00000001,
 660         0x22c9, 0xffffffff, 0x00ffffff,
 661         0xc281, 0x0000ff0f, 0x00000000,
 662         0xa293, 0x07ffffff, 0x06000000,
 663         0xf9e, 0x00000001, 0x00000002,
 664         0x31da, 0x00000008, 0x00000008,
 665         0x31dc, 0x00000f00, 0x00000800,
 666         0x31dd, 0x00000f00, 0x00000800,
 667         0x31e6, 0x00ffffff, 0x00ff7fbf,
 668         0x31e7, 0x00ffffff, 0x00ff7faf,
 669         0x2300, 0x000000ff, 0x00000800,
 670         0x390, 0x00001fff, 0x00001fff,
 671         0x2418, 0x0000007f, 0x00000020,
 672         0x2542, 0x00010000, 0x00010000,
 673         0x2b80, 0x00100000, 0x000ff07c,
 674         0x2b05, 0x000003ff, 0x0000000f,
 675         0x2b04, 0xffffffff, 0x7564fdec,
 676         0x2b03, 0xffffffff, 0x3120b9a8,
 677         0x2b02, 0x20000000, 0x0f9c0000
 678 };
 679
 680 static const u32 hawaii_mgcg_cgcg_init[] =
 681 {
 682         0x3108, 0xffffffff, 0xfffffffd,
 683         0xc200, 0xffffffff, 0xe0000000,
 684         0xf0a8, 0xffffffff, 0x00000100,
 685         0xf082, 0xffffffff, 0x00000100,
 686         0xf0b0, 0xffffffff, 0x00000100,
 687         0xf0b2, 0xffffffff, 0x00000100,
 688         0xf0b1, 0xffffffff, 0x00000100,
 689         0x1579, 0xffffffff, 0x00200100,
 690         0xf0a0, 0xffffffff, 0x00000100,
 691         0xf085, 0xffffffff, 0x06000100,
 692         0xf088, 0xffffffff, 0x00000100,
 693         0xf086, 0xffffffff, 0x06000100,
 694         0xf081, 0xffffffff, 0x00000100,
 695         0xf0b8, 0xffffffff, 0x00000100,
 696         0xf089, 0xffffffff, 0x00000100,
 697         0xf080, 0xffffffff, 0x00000100,
 698         0xf08c, 0xffffffff, 0x00000100,
 699         0xf08d, 0xffffffff, 0x00000100,
 700         0xf094, 0xffffffff, 0x00000100,
 701         0xf095, 0xffffffff, 0x00000100,
 702         0xf096, 0xffffffff, 0x00000100,
 703         0xf097, 0xffffffff, 0x00000100,
 704         0xf098, 0xffffffff, 0x00000100,
 705         0xf09f, 0xffffffff, 0x00000100,
 706         0xf09e, 0xffffffff, 0x00000100,
 707         0xf084, 0xffffffff, 0x06000100,
 708         0xf0a4, 0xffffffff, 0x00000100,
 709         0xf09d, 0xffffffff, 0x00000100,
 710         0xf0ad, 0xffffffff, 0x00000100,
 711         0xf0ac, 0xffffffff, 0x00000100,
 712         0xf09c, 0xffffffff, 0x00000100,
 713         0xc200, 0xffffffff, 0xe0000000,
 714         0xf008, 0xffffffff, 0x00010000,
 715         0xf009, 0xffffffff, 0x00030002,
 716         0xf00a, 0xffffffff, 0x00040007,
 717         0xf00b, 0xffffffff, 0x00060005,
 718         0xf00c, 0xffffffff, 0x00090008,
 719         0xf00d, 0xffffffff, 0x00010000,
 720         0xf00e, 0xffffffff, 0x00030002,
 721         0xf00f, 0xffffffff, 0x00040007,
 722         0xf010, 0xffffffff, 0x00060005,
 723         0xf011, 0xffffffff, 0x00090008,
 724         0xf012, 0xffffffff, 0x00010000,
 725         0xf013, 0xffffffff, 0x00030002,
 726         0xf014, 0xffffffff, 0x00040007,
 727         0xf015, 0xffffffff, 0x00060005,
 728         0xf016, 0xffffffff, 0x00090008,
 729         0xf017, 0xffffffff, 0x00010000,
 730         0xf018, 0xffffffff, 0x00030002,
 731         0xf019, 0xffffffff, 0x00040007,
 732         0xf01a, 0xffffffff, 0x00060005,
 733         0xf01b, 0xffffffff, 0x00090008,
 734         0xf01c, 0xffffffff, 0x00010000,
 735         0xf01d, 0xffffffff, 0x00030002,
 736         0xf01e, 0xffffffff, 0x00040007,
 737         0xf01f, 0xffffffff, 0x00060005,
 738         0xf020, 0xffffffff, 0x00090008,
 739         0xf021, 0xffffffff, 0x00010000,
 740         0xf022, 0xffffffff, 0x00030002,
 741         0xf023, 0xffffffff, 0x00040007,
 742         0xf024, 0xffffffff, 0x00060005,
 743         0xf025, 0xffffffff, 0x00090008,
 744         0xf026, 0xffffffff, 0x00010000,
 745         0xf027, 0xffffffff, 0x00030002,
 746         0xf028, 0xffffffff, 0x00040007,
 747         0xf029, 0xffffffff, 0x00060005,
 748         0xf02a, 0xffffffff, 0x00090008,
 749         0xf02b, 0xffffffff, 0x00010000,
 750         0xf02c, 0xffffffff, 0x00030002,
 751         0xf02d, 0xffffffff, 0x00040007,
 752         0xf02e, 0xffffffff, 0x00060005,
 753         0xf02f, 0xffffffff, 0x00090008,
 754         0xf030, 0xffffffff, 0x00010000,
 755         0xf031, 0xffffffff, 0x00030002,
 756         0xf032, 0xffffffff, 0x00040007,
 757         0xf033, 0xffffffff, 0x00060005,
 758         0xf034, 0xffffffff, 0x00090008,
 759         0xf035, 0xffffffff, 0x00010000,
 760         0xf036, 0xffffffff, 0x00030002,
 761         0xf037, 0xffffffff, 0x00040007,
 762         0xf038, 0xffffffff, 0x00060005,
 763         0xf039, 0xffffffff, 0x00090008,
 764         0xf03a, 0xffffffff, 0x00010000,
 765         0xf03b, 0xffffffff, 0x00030002,
 766         0xf03c, 0xffffffff, 0x00040007,
 767         0xf03d, 0xffffffff, 0x00060005,
 768         0xf03e, 0xffffffff, 0x00090008,
 769         0x30c6, 0xffffffff, 0x00020200,
 770         0xcd4, 0xffffffff, 0x00000200,
 771         0x570, 0xffffffff, 0x00000400,
 772         0x157a, 0xffffffff, 0x00000000,
 773         0xbd4, 0xffffffff, 0x00000902,
 774         0xf000, 0xffffffff, 0x96940200,
 775         0x21c2, 0xffffffff, 0x00900100,
 776         0x3109, 0xffffffff, 0x0020003f,
 777         0xe, 0xffffffff, 0x0140001c,
 778         0xf, 0x000f0000, 0x000f0000,
 779         0x88, 0xffffffff, 0xc060000c,
 780         0x89, 0xc0000fff, 0x00000100,
 781         0x3e4, 0xffffffff, 0x00000100,
 782         0x3e6, 0x00000101, 0x00000000,
 783         0x82a, 0xffffffff, 0x00000104,
 784         0x1579, 0xff000fff, 0x00000100,
 785         0xc33, 0xc0000fff, 0x00000104,
 786         0x3079, 0x00000001, 0x00000001,
 787         0x3403, 0xff000ff0, 0x00000100,
 788         0x3603, 0xff000ff0, 0x00000100
 789 };
 790
 791 static const u32 godavari_golden_registers[] =
 792 {
 793         0x1579, 0xff607fff, 0xfc000100,
 794         0x1bb6, 0x00010101, 0x00010000,
 795         0x260c, 0xffffffff, 0x00000000,
 796         0x260c0, 0xf00fffff, 0x00000400,
 797         0x184c, 0xffffffff, 0x00010000,
 798         0x16ec, 0x000000f0, 0x00000070,
 799         0x16f0, 0xf0311fff, 0x80300000,
 800         0x263e, 0x73773777, 0x12010001,
 801         0x263f, 0xffffffff, 0x00000010,
 802         0x200c, 0x00001f0f, 0x0000100a,
 803         0xbd2, 0x73773777, 0x12010001,
 804         0x902, 0x000fffff, 0x000c007f,
 805         0x2285, 0xf000003f, 0x00000007,
 806         0x22c9, 0xffffffff, 0x00ff0fff,
 807         0xc281, 0x0000ff0f, 0x00000000,
 808         0xa293, 0x07ffffff, 0x06000000,
 809         0x136, 0x00000fff, 0x00000100,
 810         0x3405, 0x00010000, 0x00810001,
 811         0x3605, 0x00010000, 0x00810001,
 812         0xf9e, 0x00000001, 0x00000002,
 813         0x31da, 0x00000008, 0x00000008,
 814         0x31dc, 0x00000f00, 0x00000800,
 815         0x31dd, 0x00000f00, 0x00000800,
 816         0x31e6, 0x00ffffff, 0x00ff7fbf,
 817         0x31e7, 0x00ffffff, 0x00ff7faf,
 818         0x2300, 0x000000ff, 0x00000001,
 819         0x853e, 0x01ff01ff, 0x00000002,
 820         0x8526, 0x007ff800, 0x00200000,
 821         0x8057, 0xffffffff, 0x00000f40,
 822         0x2231, 0x001f3ae3, 0x00000082,
 823         0x2235, 0x0000001f, 0x00000010,
 824         0xc24d, 0xffffffff, 0x00000000
 825 };
 826
 827 static void cik_init_golden_registers(struct amdgpu_device *adev)
 828 {
 829         /* Some of the registers might be dependent on GRBM_GFX_INDEX */
 830         mutex_lock(&adev->grbm_idx_mutex);
 831
 832         switch (adev->asic_type) {
 833         case CHIP_BONAIRE:
 834                 amdgpu_device_program_register_sequence(adev,
 835                                                         bonaire_mgcg_cgcg_init,
 836                                                         ARRAY_SIZE(bonaire_mgcg_cgcg_init));
 837                 amdgpu_device_program_register_sequence(adev,
 838                                                         bonaire_golden_registers,
 839                                                         ARRAY_SIZE(bonaire_golden_registers));
 840                 amdgpu_device_program_register_sequence(adev,
 841                                                         bonaire_golden_common_registers,
 842                                                         ARRAY_SIZE(bonaire_golden_common_registers));
 843                 amdgpu_device_program_register_sequence(adev,
 844                                                         bonaire_golden_spm_registers,
 845                                                         ARRAY_SIZE(bonaire_golden_spm_registers));
 846                 break;
 847         case CHIP_KABINI:
 848                 amdgpu_device_program_register_sequence(adev,
 849                                                         kalindi_mgcg_cgcg_init,
 850                                                         ARRAY_SIZE(kalindi_mgcg_cgcg_init));
 851                 amdgpu_device_program_register_sequence(adev,
 852                                                         kalindi_golden_registers,
 853                                                         ARRAY_SIZE(kalindi_golden_registers));
 854                 amdgpu_device_program_register_sequence(adev,
 855                                                         kalindi_golden_common_registers,
 856                                                         ARRAY_SIZE(kalindi_golden_common_registers));
 857                 amdgpu_device_program_register_sequence(adev,
 858                                                         kalindi_golden_spm_registers,
 859                                                         ARRAY_SIZE(kalindi_golden_spm_registers));
 860                 break;
 861         case CHIP_MULLINS:
 862                 amdgpu_device_program_register_sequence(adev,
 863                                                         kalindi_mgcg_cgcg_init,
 864                                                         ARRAY_SIZE(kalindi_mgcg_cgcg_init));
 865                 amdgpu_device_program_register_sequence(adev,
 866                                                         godavari_golden_registers,
 867                                                         ARRAY_SIZE(godavari_golden_registers));
 868                 amdgpu_device_program_register_sequence(adev,
 869                                                         kalindi_golden_common_registers,
 870                                                         ARRAY_SIZE(kalindi_golden_common_registers));
 871                 amdgpu_device_program_register_sequence(adev,
 872                                                         kalindi_golden_spm_registers,
 873                                                         ARRAY_SIZE(kalindi_golden_spm_registers));
 874                 break;
 875         case CHIP_KAVERI:
 876                 amdgpu_device_program_register_sequence(adev,
 877                                                         spectre_mgcg_cgcg_init,
 878                                                         ARRAY_SIZE(spectre_mgcg_cgcg_init));
 879                 amdgpu_device_program_register_sequence(adev,
 880                                                         spectre_golden_registers,
 881                                                         ARRAY_SIZE(spectre_golden_registers));
 882                 amdgpu_device_program_register_sequence(adev,
 883                                                         spectre_golden_common_registers,
 884                                                         ARRAY_SIZE(spectre_golden_common_registers));
 885                 amdgpu_device_program_register_sequence(adev,
 886                                                         spectre_golden_spm_registers,
 887                                                         ARRAY_SIZE(spectre_golden_spm_registers));
 888                 break;
 889         case CHIP_HAWAII:
 890                 amdgpu_device_program_register_sequence(adev,
 891                                                         hawaii_mgcg_cgcg_init,
 892                                                         ARRAY_SIZE(hawaii_mgcg_cgcg_init));
 893                 amdgpu_device_program_register_sequence(adev,
 894                                                         hawaii_golden_registers,
 895                                                         ARRAY_SIZE(hawaii_golden_registers));
 896                 amdgpu_device_program_register_sequence(adev,
 897                                                         hawaii_golden_common_registers,
 898                                                         ARRAY_SIZE(hawaii_golden_common_registers));
 899                 amdgpu_device_program_register_sequence(adev,
 900                                                         hawaii_golden_spm_registers,
 901                                                         ARRAY_SIZE(hawaii_golden_spm_registers));
 902                 break;
 903         default:
 904                 break;
 905         }
 906         mutex_unlock(&adev->grbm_idx_mutex);
 907 }
 908
 909 /**
 910  * cik_get_xclk - get the xclk
 911  *
 912  * @adev: amdgpu_device pointer
 913  *
 914  * Returns the reference clock used by the gfx engine
 915  * (CIK).
 916  */
 917 static u32 cik_get_xclk(struct amdgpu_device *adev)
 918 {
 919         u32 reference_clock = adev->clock.spll.reference_freq;
 920
 921         if (adev->flags & AMD_IS_APU) {
 922                 if (RREG32_SMC(ixGENERAL_PWRMGT) & GENERAL_PWRMGT__GPU_COUNTER_CLK_MASK)
 923                         return reference_clock / 2;
 924         } else {
 925                 if (RREG32_SMC(ixCG_CLKPIN_CNTL) & CG_CLKPIN_CNTL__XTALIN_DIVIDE_MASK)
 926                         return reference_clock / 4;
 927         }
 928         return reference_clock;
 929 }
 930
 931 /**
 932  * cik_srbm_select - select specific register instances
 933  *
 934  * @adev: amdgpu_device pointer
 935  * @me: selected ME (micro engine)
 936  * @pipe: pipe
 937  * @queue: queue
 938  * @vmid: VMID
 939  *
 940  * Switches the currently active registers instances.  Some
 941  * registers are instanced per VMID, others are instanced per
 942  * me/pipe/queue combination.
 943  */
 944 void cik_srbm_select(struct amdgpu_device *adev,
 945                      u32 me, u32 pipe, u32 queue, u32 vmid)
 946 {
 947         u32 srbm_gfx_cntl =
 948                 (((pipe << SRBM_GFX_CNTL__PIPEID__SHIFT) & SRBM_GFX_CNTL__PIPEID_MASK)|
 949                 ((me << SRBM_GFX_CNTL__MEID__SHIFT) & SRBM_GFX_CNTL__MEID_MASK)|
 950                 ((vmid << SRBM_GFX_CNTL__VMID__SHIFT) & SRBM_GFX_CNTL__VMID_MASK)|
 951                 ((queue << SRBM_GFX_CNTL__QUEUEID__SHIFT) & SRBM_GFX_CNTL__QUEUEID_MASK));
 952         WREG32(mmSRBM_GFX_CNTL, srbm_gfx_cntl);
 953 }
 954
 955 static void cik_vga_set_state(struct amdgpu_device *adev, bool state)
 956 {
 957         uint32_t tmp;
 958
 959         tmp = RREG32(mmCONFIG_CNTL);
 960         if (!state)
 961                 tmp |= CONFIG_CNTL__VGA_DIS_MASK;
 962         else
 963                 tmp &= ~CONFIG_CNTL__VGA_DIS_MASK;
 964         WREG32(mmCONFIG_CNTL, tmp);
 965 }
 966
 967 static bool cik_read_disabled_bios(struct amdgpu_device *adev)
 968 {
 969         u32 bus_cntl;
 970         u32 d1vga_control = 0;
 971         u32 d2vga_control = 0;
 972         u32 vga_render_control = 0;
 973         u32 rom_cntl;
 974         bool r;
 975
 976         bus_cntl = RREG32(mmBUS_CNTL);
 977         if (adev->mode_info.num_crtc) {
 978                 d1vga_control = RREG32(mmD1VGA_CONTROL);
 979                 d2vga_control = RREG32(mmD2VGA_CONTROL);
 980                 vga_render_control = RREG32(mmVGA_RENDER_CONTROL);
 981         }
 982         rom_cntl = RREG32_SMC(ixROM_CNTL);
 983
 984         /* enable the rom */
 985         WREG32(mmBUS_CNTL, (bus_cntl & ~BUS_CNTL__BIOS_ROM_DIS_MASK));
 986         if (adev->mode_info.num_crtc) {
 987                 /* Disable VGA mode */
 988                 WREG32(mmD1VGA_CONTROL,
 989                        (d1vga_control & ~(D1VGA_CONTROL__D1VGA_MODE_ENABLE_MASK |
 990                                           D1VGA_CONTROL__D1VGA_TIMING_SELECT_MASK)));
 991                 WREG32(mmD2VGA_CONTROL,
 992                        (d2vga_control & ~(D1VGA_CONTROL__D1VGA_MODE_ENABLE_MASK |
 993                                           D1VGA_CONTROL__D1VGA_TIMING_SELECT_MASK)));
 994                 WREG32(mmVGA_RENDER_CONTROL,
 995                        (vga_render_control & ~VGA_RENDER_CONTROL__VGA_VSTATUS_CNTL_MASK));
 996         }
 997         WREG32_SMC(ixROM_CNTL, rom_cntl | ROM_CNTL__SCK_OVERWRITE_MASK);
 998
 999         r = amdgpu_read_bios(adev);
1000
1001         /* restore regs */
1002         WREG32(mmBUS_CNTL, bus_cntl);
1003         if (adev->mode_info.num_crtc) {
1004                 WREG32(mmD1VGA_CONTROL, d1vga_control);
1005                 WREG32(mmD2VGA_CONTROL, d2vga_control);
1006                 WREG32(mmVGA_RENDER_CONTROL, vga_render_control);
1007         }
1008         WREG32_SMC(ixROM_CNTL, rom_cntl);
1009         return r;
1010 }
1011
1012 static bool cik_read_bios_from_rom(struct amdgpu_device *adev,
1013                                    u8 *bios, u32 length_bytes)
1014 {
1015         u32 *dw_ptr;
1016         unsigned long flags;
1017         u32 i, length_dw;
1018
1019         if (bios == NULL)
1020                 return false;
1021         if (length_bytes == 0)
1022                 return false;
1023         /* APU vbios image is part of sbios image */
1024         if (adev->flags & AMD_IS_APU)
1025                 return false;
1026
1027         dw_ptr = (u32 *)bios;
1028         length_dw = ALIGN(length_bytes, 4) / 4;
1029         /* take the smc lock since we are using the smc index */
1030         spin_lock_irqsave(&adev->smc_idx_lock, flags);
1031         /* set rom index to 0 */
1032         WREG32(mmSMC_IND_INDEX_0, ixROM_INDEX);
1033         WREG32(mmSMC_IND_DATA_0, 0);
1034         /* set index to data for continous read */
1035         WREG32(mmSMC_IND_INDEX_0, ixROM_DATA);
1036         for (i = 0; i < length_dw; i++)
1037                 dw_ptr[i] = RREG32(mmSMC_IND_DATA_0);
1038         spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
1039
1040         return true;
1041 }
1042
1043 static const struct amdgpu_allowed_register_entry cik_allowed_read_registers[] = {
1044         {mmGRBM_STATUS},
1045         {mmGRBM_STATUS2},
1046         {mmGRBM_STATUS_SE0},
1047         {mmGRBM_STATUS_SE1},
1048         {mmGRBM_STATUS_SE2},
1049         {mmGRBM_STATUS_SE3},
1050         {mmSRBM_STATUS},
1051         {mmSRBM_STATUS2},
1052         {mmSDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET},
1053         {mmSDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET},
1054         {mmCP_STAT},
1055         {mmCP_STALLED_STAT1},
1056         {mmCP_STALLED_STAT2},
1057         {mmCP_STALLED_STAT3},
1058         {mmCP_CPF_BUSY_STAT},
1059         {mmCP_CPF_STALLED_STAT1},
1060         {mmCP_CPF_STATUS},
1061         {mmCP_CPC_BUSY_STAT},
1062         {mmCP_CPC_STALLED_STAT1},
1063         {mmCP_CPC_STATUS},
1064         {mmGB_ADDR_CONFIG},
1065         {mmMC_ARB_RAMCFG},
1066         {mmGB_TILE_MODE0},
1067         {mmGB_TILE_MODE1},
1068         {mmGB_TILE_MODE2},
1069         {mmGB_TILE_MODE3},
1070         {mmGB_TILE_MODE4},
1071         {mmGB_TILE_MODE5},
1072         {mmGB_TILE_MODE6},
1073         {mmGB_TILE_MODE7},
1074         {mmGB_TILE_MODE8},
1075         {mmGB_TILE_MODE9},
1076         {mmGB_TILE_MODE10},
1077         {mmGB_TILE_MODE11},
1078         {mmGB_TILE_MODE12},
1079         {mmGB_TILE_MODE13},
1080         {mmGB_TILE_MODE14},
1081         {mmGB_TILE_MODE15},
1082         {mmGB_TILE_MODE16},
1083         {mmGB_TILE_MODE17},
1084         {mmGB_TILE_MODE18},
1085         {mmGB_TILE_MODE19},
1086         {mmGB_TILE_MODE20},
1087         {mmGB_TILE_MODE21},
1088         {mmGB_TILE_MODE22},
1089         {mmGB_TILE_MODE23},
1090         {mmGB_TILE_MODE24},
1091         {mmGB_TILE_MODE25},
1092         {mmGB_TILE_MODE26},
1093         {mmGB_TILE_MODE27},
1094         {mmGB_TILE_MODE28},
1095         {mmGB_TILE_MODE29},
1096         {mmGB_TILE_MODE30},
1097         {mmGB_TILE_MODE31},
1098         {mmGB_MACROTILE_MODE0},
1099         {mmGB_MACROTILE_MODE1},
1100         {mmGB_MACROTILE_MODE2},
1101         {mmGB_MACROTILE_MODE3},
1102         {mmGB_MACROTILE_MODE4},
1103         {mmGB_MACROTILE_MODE5},
1104         {mmGB_MACROTILE_MODE6},
1105         {mmGB_MACROTILE_MODE7},
1106         {mmGB_MACROTILE_MODE8},
1107         {mmGB_MACROTILE_MODE9},
1108         {mmGB_MACROTILE_MODE10},
1109         {mmGB_MACROTILE_MODE11},
1110         {mmGB_MACROTILE_MODE12},
1111         {mmGB_MACROTILE_MODE13},
1112         {mmGB_MACROTILE_MODE14},
1113         {mmGB_MACROTILE_MODE15},
1114         {mmCC_RB_BACKEND_DISABLE, true},
1115         {mmGC_USER_RB_BACKEND_DISABLE, true},
1116         {mmGB_BACKEND_MAP, false},
1117         {mmPA_SC_RASTER_CONFIG, true},
1118         {mmPA_SC_RASTER_CONFIG_1, true},
1119 };
1120
1121
1122 static uint32_t cik_get_register_value(struct amdgpu_device *adev,
1123                                        bool indexed, u32 se_num,
1124                                        u32 sh_num, u32 reg_offset)
1125 {
1126         if (indexed) {
1127                 uint32_t val;
1128                 unsigned se_idx = (se_num == 0xffffffff) ? 0 : se_num;
1129                 unsigned sh_idx = (sh_num == 0xffffffff) ? 0 : sh_num;
1130
1131                 switch (reg_offset) {
1132                 case mmCC_RB_BACKEND_DISABLE:
1133                         return adev->gfx.config.rb_config[se_idx][sh_idx].rb_backend_disable;
1134                 case mmGC_USER_RB_BACKEND_DISABLE:
1135                         return adev->gfx.config.rb_config[se_idx][sh_idx].user_rb_backend_disable;
1136                 case mmPA_SC_RASTER_CONFIG:
1137                         return adev->gfx.config.rb_config[se_idx][sh_idx].raster_config;
1138                 case mmPA_SC_RASTER_CONFIG_1:
1139                         return adev->gfx.config.rb_config[se_idx][sh_idx].raster_config_1;
1140                 }
1141
1142                 mutex_lock(&adev->grbm_idx_mutex);
1143                 if (se_num != 0xffffffff || sh_num != 0xffffffff)
1144                         amdgpu_gfx_select_se_sh(adev, se_num, sh_num, 0xffffffff, 0);
1145
1146                 val = RREG32(reg_offset);
1147
1148                 if (se_num != 0xffffffff || sh_num != 0xffffffff)
1149                         amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
1150                 mutex_unlock(&adev->grbm_idx_mutex);
1151                 return val;
1152         } else {
1153                 unsigned idx;
1154
1155                 switch (reg_offset) {
1156                 case mmGB_ADDR_CONFIG:
1157                         return adev->gfx.config.gb_addr_config;
1158                 case mmMC_ARB_RAMCFG:
1159                         return adev->gfx.config.mc_arb_ramcfg;
1160                 case mmGB_TILE_MODE0:
1161                 case mmGB_TILE_MODE1:
1162                 case mmGB_TILE_MODE2:
1163                 case mmGB_TILE_MODE3:
1164                 case mmGB_TILE_MODE4:
1165                 case mmGB_TILE_MODE5:
1166                 case mmGB_TILE_MODE6:
1167                 case mmGB_TILE_MODE7:
1168                 case mmGB_TILE_MODE8:
1169                 case mmGB_TILE_MODE9:
1170                 case mmGB_TILE_MODE10:
1171                 case mmGB_TILE_MODE11:
1172                 case mmGB_TILE_MODE12:
1173                 case mmGB_TILE_MODE13:
1174                 case mmGB_TILE_MODE14:
1175                 case mmGB_TILE_MODE15:
1176                 case mmGB_TILE_MODE16:
1177                 case mmGB_TILE_MODE17:
1178                 case mmGB_TILE_MODE18:
1179                 case mmGB_TILE_MODE19:
1180                 case mmGB_TILE_MODE20:
1181                 case mmGB_TILE_MODE21:
1182                 case mmGB_TILE_MODE22:
1183                 case mmGB_TILE_MODE23:
1184                 case mmGB_TILE_MODE24:
1185                 case mmGB_TILE_MODE25:
1186                 case mmGB_TILE_MODE26:
1187                 case mmGB_TILE_MODE27:
1188                 case mmGB_TILE_MODE28:
1189                 case mmGB_TILE_MODE29:
1190                 case mmGB_TILE_MODE30:
1191                 case mmGB_TILE_MODE31:
1192                         idx = (reg_offset - mmGB_TILE_MODE0);
1193                         return adev->gfx.config.tile_mode_array[idx];
1194                 case mmGB_MACROTILE_MODE0:
1195                 case mmGB_MACROTILE_MODE1:
1196                 case mmGB_MACROTILE_MODE2:
1197                 case mmGB_MACROTILE_MODE3:
1198                 case mmGB_MACROTILE_MODE4:
1199                 case mmGB_MACROTILE_MODE5:
1200                 case mmGB_MACROTILE_MODE6:
1201                 case mmGB_MACROTILE_MODE7:
1202                 case mmGB_MACROTILE_MODE8:
1203                 case mmGB_MACROTILE_MODE9:
1204                 case mmGB_MACROTILE_MODE10:
1205                 case mmGB_MACROTILE_MODE11:
1206                 case mmGB_MACROTILE_MODE12:
1207                 case mmGB_MACROTILE_MODE13:
1208                 case mmGB_MACROTILE_MODE14:
1209                 case mmGB_MACROTILE_MODE15:
1210                         idx = (reg_offset - mmGB_MACROTILE_MODE0);
1211                         return adev->gfx.config.macrotile_mode_array[idx];
1212                 default:
1213                         return RREG32(reg_offset);
1214                 }
1215         }
1216 }
1217
1218 static int cik_read_register(struct amdgpu_device *adev, u32 se_num,
1219                              u32 sh_num, u32 reg_offset, u32 *value)
1220 {
1221         uint32_t i;
1222
1223         *value = 0;
1224         for (i = 0; i < ARRAY_SIZE(cik_allowed_read_registers); i++) {
1225                 bool indexed = cik_allowed_read_registers[i].grbm_indexed;
1226
1227                 if (reg_offset != cik_allowed_read_registers[i].reg_offset)
1228                         continue;
1229
1230                 *value = cik_get_register_value(adev, indexed, se_num, sh_num,
1231                                                 reg_offset);
1232                 return 0;
1233         }
1234         return -EINVAL;
1235 }
1236
1237 struct kv_reset_save_regs {
1238         u32 gmcon_reng_execute;
1239         u32 gmcon_misc;
1240         u32 gmcon_misc3;
1241 };
1242
1243 static void kv_save_regs_for_reset(struct amdgpu_device *adev,
1244                                    struct kv_reset_save_regs *save)
1245 {
1246         save->gmcon_reng_execute = RREG32(mmGMCON_RENG_EXECUTE);
1247         save->gmcon_misc = RREG32(mmGMCON_MISC);
1248         save->gmcon_misc3 = RREG32(mmGMCON_MISC3);
1249
1250         WREG32(mmGMCON_RENG_EXECUTE, save->gmcon_reng_execute &
1251                 ~GMCON_RENG_EXECUTE__RENG_EXECUTE_ON_PWR_UP_MASK);
1252         WREG32(mmGMCON_MISC, save->gmcon_misc &
1253                 ~(GMCON_MISC__RENG_EXECUTE_ON_REG_UPDATE_MASK |
1254                         GMCON_MISC__STCTRL_STUTTER_EN_MASK));
1255 }
1256
1257 static void kv_restore_regs_for_reset(struct amdgpu_device *adev,
1258                                       struct kv_reset_save_regs *save)
1259 {
1260         int i;
1261
1262         WREG32(mmGMCON_PGFSM_WRITE, 0);
1263         WREG32(mmGMCON_PGFSM_CONFIG, 0x200010ff);
1264
1265         for (i = 0; i < 5; i++)
1266                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1267
1268         WREG32(mmGMCON_PGFSM_WRITE, 0);
1269         WREG32(mmGMCON_PGFSM_CONFIG, 0x300010ff);
1270
1271         for (i = 0; i < 5; i++)
1272                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1273
1274         WREG32(mmGMCON_PGFSM_WRITE, 0x210000);
1275         WREG32(mmGMCON_PGFSM_CONFIG, 0xa00010ff);
1276
1277         for (i = 0; i < 5; i++)
1278                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1279
1280         WREG32(mmGMCON_PGFSM_WRITE, 0x21003);
1281         WREG32(mmGMCON_PGFSM_CONFIG, 0xb00010ff);
1282
1283         for (i = 0; i < 5; i++)
1284                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1285
1286         WREG32(mmGMCON_PGFSM_WRITE, 0x2b00);
1287         WREG32(mmGMCON_PGFSM_CONFIG, 0xc00010ff);
1288
1289         for (i = 0; i < 5; i++)
1290                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1291
1292         WREG32(mmGMCON_PGFSM_WRITE, 0);
1293         WREG32(mmGMCON_PGFSM_CONFIG, 0xd00010ff);
1294
1295         for (i = 0; i < 5; i++)
1296                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1297
1298         WREG32(mmGMCON_PGFSM_WRITE, 0x420000);
1299         WREG32(mmGMCON_PGFSM_CONFIG, 0x100010ff);
1300
1301         for (i = 0; i < 5; i++)
1302                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1303
1304         WREG32(mmGMCON_PGFSM_WRITE, 0x120202);
1305         WREG32(mmGMCON_PGFSM_CONFIG, 0x500010ff);
1306
1307         for (i = 0; i < 5; i++)
1308                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1309
1310         WREG32(mmGMCON_PGFSM_WRITE, 0x3e3e36);
1311         WREG32(mmGMCON_PGFSM_CONFIG, 0x600010ff);
1312
1313         for (i = 0; i < 5; i++)
1314                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1315
1316         WREG32(mmGMCON_PGFSM_WRITE, 0x373f3e);
1317         WREG32(mmGMCON_PGFSM_CONFIG, 0x700010ff);
1318
1319         for (i = 0; i < 5; i++)
1320                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1321
1322         WREG32(mmGMCON_PGFSM_WRITE, 0x3e1332);
1323         WREG32(mmGMCON_PGFSM_CONFIG, 0xe00010ff);
1324
1325         WREG32(mmGMCON_MISC3, save->gmcon_misc3);
1326         WREG32(mmGMCON_MISC, save->gmcon_misc);
1327         WREG32(mmGMCON_RENG_EXECUTE, save->gmcon_reng_execute);
1328 }
1329
1330 /**
1331  * cik_asic_pci_config_reset - soft reset GPU
1332  *
1333  * @adev: amdgpu_device pointer
1334  *
1335  * Use PCI Config method to reset the GPU.
1336  *
1337  * Returns 0 for success.
1338  */
1339 static int cik_asic_pci_config_reset(struct amdgpu_device *adev)
1340 {
1341         struct kv_reset_save_regs kv_save = { 0 };
1342         u32 i;
1343         int r = -EINVAL;
1344
1345         amdgpu_atombios_scratch_regs_engine_hung(adev, true);
1346
1347         if (adev->flags & AMD_IS_APU)
1348                 kv_save_regs_for_reset(adev, &kv_save);
1349
1350         /* disable BM */
1351         pci_clear_master(adev->pdev);
1352         /* reset */
1353         amdgpu_device_pci_config_reset(adev);
1354
1355         udelay(100);
1356
1357         /* wait for asic to come out of reset */
1358         for (i = 0; i < adev->usec_timeout; i++) {
1359                 if (RREG32(mmCONFIG_MEMSIZE) != 0xffffffff) {
1360                         /* enable BM */
1361                         pci_set_master(adev->pdev);
1362                         adev->has_hw_reset = true;
1363                         r = 0;
1364                         break;
1365                 }
1366                 udelay(1);
1367         }
1368
1369         /* does asic init need to be run first??? */
1370         if (adev->flags & AMD_IS_APU)
1371                 kv_restore_regs_for_reset(adev, &kv_save);
1372
1373         amdgpu_atombios_scratch_regs_engine_hung(adev, false);
1374
1375         return r;
1376 }
1377
1378 static int cik_asic_supports_baco(struct amdgpu_device *adev)
1379 {
1380         switch (adev->asic_type) {
1381         case CHIP_BONAIRE:
1382         case CHIP_HAWAII:
1383                 return amdgpu_dpm_is_baco_supported(adev);
1384         default:
1385                 return 0;
1386         }
1387 }
1388
1389 static enum amd_reset_method
1390 cik_asic_reset_method(struct amdgpu_device *adev)
1391 {
1392         bool baco_reset;
1393
1394         if (amdgpu_reset_method == AMD_RESET_METHOD_LEGACY ||
1395             amdgpu_reset_method == AMD_RESET_METHOD_BACO)
1396                 return amdgpu_reset_method;
1397
1398         if (amdgpu_reset_method != -1)
1399                 dev_warn(adev->dev, "Specified reset:%d isn't supported, using AUTO instead.\n",
1400                                   amdgpu_reset_method);
1401
1402         switch (adev->asic_type) {
1403         case CHIP_BONAIRE:
1404         case CHIP_HAWAII:
1405                 baco_reset = cik_asic_supports_baco(adev);
1406                 break;
1407         default:
1408                 baco_reset = false;
1409                 break;
1410         }
1411
1412         if (baco_reset)
1413                 return AMD_RESET_METHOD_BACO;
1414         else
1415                 return AMD_RESET_METHOD_LEGACY;
1416 }
1417
1418 /**
1419  * cik_asic_reset - soft reset GPU
1420  *
1421  * @adev: amdgpu_device pointer
1422  *
1423  * Look up which blocks are hung and attempt
1424  * to reset them.
1425  * Returns 0 for success.
1426  */
1427 static int cik_asic_reset(struct amdgpu_device *adev)
1428 {
1429         int r;
1430
1431         /* APUs don't have full asic reset */
1432         if (adev->flags & AMD_IS_APU)
1433                 return 0;
1434
1435         if (cik_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) {
1436                 dev_info(adev->dev, "BACO reset\n");
1437                 r = amdgpu_dpm_baco_reset(adev);
1438         } else {
1439                 dev_info(adev->dev, "PCI CONFIG reset\n");
1440                 r = cik_asic_pci_config_reset(adev);
1441         }
1442
1443         return r;
1444 }
1445
1446 static u32 cik_get_config_memsize(struct amdgpu_device *adev)
1447 {
1448         return RREG32(mmCONFIG_MEMSIZE);
1449 }
1450
1451 static int cik_set_uvd_clock(struct amdgpu_device *adev, u32 clock,
1452                               u32 cntl_reg, u32 status_reg)
1453 {
1454         int r, i;
1455         struct atom_clock_dividers dividers;
1456         uint32_t tmp;
1457
1458         r = amdgpu_atombios_get_clock_dividers(adev,
1459                                                COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
1460                                                clock, false, &dividers);
1461         if (r)
1462                 return r;
1463
1464         tmp = RREG32_SMC(cntl_reg);
1465         tmp &= ~(CG_DCLK_CNTL__DCLK_DIR_CNTL_EN_MASK |
1466                 CG_DCLK_CNTL__DCLK_DIVIDER_MASK);
1467         tmp |= dividers.post_divider;
1468         WREG32_SMC(cntl_reg, tmp);
1469
1470         for (i = 0; i < 100; i++) {
1471                 if (RREG32_SMC(status_reg) & CG_DCLK_STATUS__DCLK_STATUS_MASK)
1472                         break;
1473                 mdelay(10);
1474         }
1475         if (i == 100)
1476                 return -ETIMEDOUT;
1477
1478         return 0;
1479 }
1480
1481 static int cik_set_uvd_clocks(struct amdgpu_device *adev, u32 vclk, u32 dclk)
1482 {
1483         int r = 0;
1484
1485         r = cik_set_uvd_clock(adev, vclk, ixCG_VCLK_CNTL, ixCG_VCLK_STATUS);
1486         if (r)
1487                 return r;
1488
1489         r = cik_set_uvd_clock(adev, dclk, ixCG_DCLK_CNTL, ixCG_DCLK_STATUS);
1490         return r;
1491 }
1492
1493 static int cik_set_vce_clocks(struct amdgpu_device *adev, u32 evclk, u32 ecclk)
1494 {
1495         int r, i;
1496         struct atom_clock_dividers dividers;
1497         u32 tmp;
1498
1499         r = amdgpu_atombios_get_clock_dividers(adev,
1500                                                COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
1501                                                ecclk, false, &dividers);
1502         if (r)
1503                 return r;
1504
1505         for (i = 0; i < 100; i++) {
1506                 if (RREG32_SMC(ixCG_ECLK_STATUS) & CG_ECLK_STATUS__ECLK_STATUS_MASK)
1507                         break;
1508                 mdelay(10);
1509         }
1510         if (i == 100)
1511                 return -ETIMEDOUT;
1512
1513         tmp = RREG32_SMC(ixCG_ECLK_CNTL);
1514         tmp &= ~(CG_ECLK_CNTL__ECLK_DIR_CNTL_EN_MASK |
1515                 CG_ECLK_CNTL__ECLK_DIVIDER_MASK);
1516         tmp |= dividers.post_divider;
1517         WREG32_SMC(ixCG_ECLK_CNTL, tmp);
1518
1519         for (i = 0; i < 100; i++) {
1520                 if (RREG32_SMC(ixCG_ECLK_STATUS) & CG_ECLK_STATUS__ECLK_STATUS_MASK)
1521                         break;
1522                 mdelay(10);
1523         }
1524         if (i == 100)
1525                 return -ETIMEDOUT;
1526
1527         return 0;
1528 }
1529
1530 static void cik_pcie_gen3_enable(struct amdgpu_device *adev)
1531 {
1532         struct pci_dev *root = adev->pdev->bus->self;
1533         u32 speed_cntl, current_data_rate;
1534         int i;
1535         u16 tmp16;
1536
1537         if (pci_is_root_bus(adev->pdev->bus))
1538                 return;
1539
1540         if (amdgpu_pcie_gen2 == 0)
1541                 return;
1542
1543         if (adev->flags & AMD_IS_APU)
1544                 return;
1545
1546         if (!(adev->pm.pcie_gen_mask & (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
1547                                         CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)))
1548                 return;
1549
1550         speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1551         current_data_rate = (speed_cntl & PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE_MASK) >>
1552                 PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE__SHIFT;
1553         if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
1554                 if (current_data_rate == 2) {
1555                         DRM_INFO("PCIE gen 3 link speeds already enabled\n");
1556                         return;
1557                 }
1558                 DRM_INFO("enabling PCIE gen 3 link speeds, disable with amdgpu.pcie_gen2=0\n");
1559         } else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2) {
1560                 if (current_data_rate == 1) {
1561                         DRM_INFO("PCIE gen 2 link speeds already enabled\n");
1562                         return;
1563                 }
1564                 DRM_INFO("enabling PCIE gen 2 link speeds, disable with amdgpu.pcie_gen2=0\n");
1565         }
1566
1567         if (!pci_is_pcie(root) || !pci_is_pcie(adev->pdev))
1568                 return;
1569
1570         if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
1571                 /* re-try equalization if gen3 is not already enabled */
1572                 if (current_data_rate != 2) {
1573                         u16 bridge_cfg, gpu_cfg;
1574                         u16 bridge_cfg2, gpu_cfg2;
1575                         u32 max_lw, current_lw, tmp;
1576
1577                         pcie_capability_set_word(root, PCI_EXP_LNKCTL, PCI_EXP_LNKCTL_HAWD);
1578                         pcie_capability_set_word(adev->pdev, PCI_EXP_LNKCTL, PCI_EXP_LNKCTL_HAWD);
1579
1580                         tmp = RREG32_PCIE(ixPCIE_LC_STATUS1);
1581                         max_lw = (tmp & PCIE_LC_STATUS1__LC_DETECTED_LINK_WIDTH_MASK) >>
1582                                 PCIE_LC_STATUS1__LC_DETECTED_LINK_WIDTH__SHIFT;
1583                         current_lw = (tmp & PCIE_LC_STATUS1__LC_OPERATING_LINK_WIDTH_MASK)
1584                                 >> PCIE_LC_STATUS1__LC_OPERATING_LINK_WIDTH__SHIFT;
1585
1586                         if (current_lw < max_lw) {
1587                                 tmp = RREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL);
1588                                 if (tmp & PCIE_LC_LINK_WIDTH_CNTL__LC_RENEGOTIATION_SUPPORT_MASK) {
1589                                         tmp &= ~(PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH_MASK |
1590                                                 PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_DIS_MASK);
1591                                         tmp |= (max_lw <<
1592                                                 PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH__SHIFT);
1593                                         tmp |= PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_SUPPORT_MASK |
1594                                         PCIE_LC_LINK_WIDTH_CNTL__LC_RENEGOTIATE_EN_MASK |
1595                                         PCIE_LC_LINK_WIDTH_CNTL__LC_RECONFIG_NOW_MASK;
1596                                         WREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL, tmp);
1597                                 }
1598                         }
1599
1600                         for (i = 0; i < 10; i++) {
1601                                 /* check status */
1602                                 pcie_capability_read_word(adev->pdev,
1603                                                           PCI_EXP_DEVSTA,
1604                                                           &tmp16);
1605                                 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
1606                                         break;
1607
1608                                 pcie_capability_read_word(root, PCI_EXP_LNKCTL,
1609                                                           &bridge_cfg);
1610                                 pcie_capability_read_word(adev->pdev,
1611                                                           PCI_EXP_LNKCTL,
1612                                                           &gpu_cfg);
1613
1614                                 pcie_capability_read_word(root, PCI_EXP_LNKCTL2,
1615                                                           &bridge_cfg2);
1616                                 pcie_capability_read_word(adev->pdev,
1617                                                           PCI_EXP_LNKCTL2,
1618                                                           &gpu_cfg2);
1619
1620                                 tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1621                                 tmp |= PCIE_LC_CNTL4__LC_SET_QUIESCE_MASK;
1622                                 WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1623
1624                                 tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1625                                 tmp |= PCIE_LC_CNTL4__LC_REDO_EQ_MASK;
1626                                 WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1627
1628                                 msleep(100);
1629
1630                                 /* linkctl */
1631                                 pcie_capability_clear_and_set_word(root, PCI_EXP_LNKCTL,
1632                                                                    PCI_EXP_LNKCTL_HAWD,
1633                                                                    bridge_cfg &
1634                                                                    PCI_EXP_LNKCTL_HAWD);
1635                                 pcie_capability_clear_and_set_word(adev->pdev, PCI_EXP_LNKCTL,
1636                                                                    PCI_EXP_LNKCTL_HAWD,
1637                                                                    gpu_cfg &
1638                                                                    PCI_EXP_LNKCTL_HAWD);
1639
1640                                 /* linkctl2 */
1641                                 pcie_capability_clear_and_set_word(root, PCI_EXP_LNKCTL2,
1642                                                                    PCI_EXP_LNKCTL2_ENTER_COMP |
1643                                                                    PCI_EXP_LNKCTL2_TX_MARGIN,
1644                                                                    bridge_cfg2 &
1645                                                                    (PCI_EXP_LNKCTL2_ENTER_COMP |
1646                                                                     PCI_EXP_LNKCTL2_TX_MARGIN));
1647                                 pcie_capability_clear_and_set_word(adev->pdev, PCI_EXP_LNKCTL2,
1648                                                                    PCI_EXP_LNKCTL2_ENTER_COMP |
1649                                                                    PCI_EXP_LNKCTL2_TX_MARGIN,
1650                                                                    gpu_cfg2 &
1651                                                                    (PCI_EXP_LNKCTL2_ENTER_COMP |
1652                                                                     PCI_EXP_LNKCTL2_TX_MARGIN));
1653
1654                                 tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1655                                 tmp &= ~PCIE_LC_CNTL4__LC_SET_QUIESCE_MASK;
1656                                 WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1657                         }
1658                 }
1659         }
1660
1661         /* set the link speed */
1662         speed_cntl |= PCIE_LC_SPEED_CNTL__LC_FORCE_EN_SW_SPEED_CHANGE_MASK |
1663                 PCIE_LC_SPEED_CNTL__LC_FORCE_DIS_HW_SPEED_CHANGE_MASK;
1664         speed_cntl &= ~PCIE_LC_SPEED_CNTL__LC_FORCE_DIS_SW_SPEED_CHANGE_MASK;
1665         WREG32_PCIE(ixPCIE_LC_SPEED_CNTL, speed_cntl);
1666
1667         tmp16 = 0;
1668         if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)
1669                 tmp16 |= PCI_EXP_LNKCTL2_TLS_8_0GT; /* gen3 */
1670         else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2)
1671                 tmp16 |= PCI_EXP_LNKCTL2_TLS_5_0GT; /* gen2 */
1672         else
1673                 tmp16 |= PCI_EXP_LNKCTL2_TLS_2_5GT; /* gen1 */
1674         pcie_capability_clear_and_set_word(adev->pdev, PCI_EXP_LNKCTL2,
1675                                            PCI_EXP_LNKCTL2_TLS, tmp16);
1676
1677         speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1678         speed_cntl |= PCIE_LC_SPEED_CNTL__LC_INITIATE_LINK_SPEED_CHANGE_MASK;
1679         WREG32_PCIE(ixPCIE_LC_SPEED_CNTL, speed_cntl);
1680
1681         for (i = 0; i < adev->usec_timeout; i++) {
1682                 speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1683                 if ((speed_cntl & PCIE_LC_SPEED_CNTL__LC_INITIATE_LINK_SPEED_CHANGE_MASK) == 0)
1684                         break;
1685                 udelay(1);
1686         }
1687 }
1688
1689 static void cik_program_aspm(struct amdgpu_device *adev)
1690 {
1691         u32 data, orig;
1692         bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
1693         bool disable_clkreq = false;
1694
1695         if (!amdgpu_device_should_use_aspm(adev))
1696                 return;
1697
1698         if (pci_is_root_bus(adev->pdev->bus))
1699                 return;
1700
1701         orig = data = RREG32_PCIE(ixPCIE_LC_N_FTS_CNTL);
1702         data &= ~PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_MASK;
1703         data |= (0x24 << PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS__SHIFT) |
1704                 PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_OVERRIDE_EN_MASK;
1705         if (orig != data)
1706                 WREG32_PCIE(ixPCIE_LC_N_FTS_CNTL, data);
1707
1708         orig = data = RREG32_PCIE(ixPCIE_LC_CNTL3);
1709         data |= PCIE_LC_CNTL3__LC_GO_TO_RECOVERY_MASK;
1710         if (orig != data)
1711                 WREG32_PCIE(ixPCIE_LC_CNTL3, data);
1712
1713         orig = data = RREG32_PCIE(ixPCIE_P_CNTL);
1714         data |= PCIE_P_CNTL__P_IGNORE_EDB_ERR_MASK;
1715         if (orig != data)
1716                 WREG32_PCIE(ixPCIE_P_CNTL, data);
1717
1718         orig = data = RREG32_PCIE(ixPCIE_LC_CNTL);
1719         data &= ~(PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK |
1720                 PCIE_LC_CNTL__LC_L1_INACTIVITY_MASK);
1721         data |= PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK;
1722         if (!disable_l0s)
1723                 data |= (7 << PCIE_LC_CNTL__LC_L0S_INACTIVITY__SHIFT);
1724
1725         if (!disable_l1) {
1726                 data |= (7 << PCIE_LC_CNTL__LC_L1_INACTIVITY__SHIFT);
1727                 data &= ~PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK;
1728                 if (orig != data)
1729                         WREG32_PCIE(ixPCIE_LC_CNTL, data);
1730
1731                 if (!disable_plloff_in_l1) {
1732                         bool clk_req_support;
1733
1734                         orig = data = RREG32_PCIE(ixPB0_PIF_PWRDOWN_0);
1735                         data &= ~(PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0_MASK |
1736                                 PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0_MASK);
1737                         data |= (7 << PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0__SHIFT) |
1738                                 (7 << PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0__SHIFT);
1739                         if (orig != data)
1740                                 WREG32_PCIE(ixPB0_PIF_PWRDOWN_0, data);
1741
1742                         orig = data = RREG32_PCIE(ixPB0_PIF_PWRDOWN_1);
1743                         data &= ~(PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1_MASK |
1744                                 PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1_MASK);
1745                         data |= (7 << PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1__SHIFT) |
1746                                 (7 << PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1__SHIFT);
1747                         if (orig != data)
1748                                 WREG32_PCIE(ixPB0_PIF_PWRDOWN_1, data);
1749
1750                         orig = data = RREG32_PCIE(ixPB1_PIF_PWRDOWN_0);
1751                         data &= ~(PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0_MASK |
1752                                 PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0_MASK);
1753                         data |= (7 << PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0__SHIFT) |
1754                                 (7 << PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0__SHIFT);
1755                         if (orig != data)
1756                                 WREG32_PCIE(ixPB1_PIF_PWRDOWN_0, data);
1757
1758                         orig = data = RREG32_PCIE(ixPB1_PIF_PWRDOWN_1);
1759                         data &= ~(PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1_MASK |
1760                                 PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1_MASK);
1761                         data |= (7 << PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1__SHIFT) |
1762                                 (7 << PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1__SHIFT);
1763                         if (orig != data)
1764                                 WREG32_PCIE(ixPB1_PIF_PWRDOWN_1, data);
1765
1766                         orig = data = RREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL);
1767                         data &= ~PCIE_LC_LINK_WIDTH_CNTL__LC_DYN_LANES_PWR_STATE_MASK;
1768                         data |= ~(3 << PCIE_LC_LINK_WIDTH_CNTL__LC_DYN_LANES_PWR_STATE__SHIFT);
1769                         if (orig != data)
1770                                 WREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL, data);
1771
1772                         if (!disable_clkreq) {
1773                                 struct pci_dev *root = adev->pdev->bus->self;
1774                                 u32 lnkcap;
1775
1776                                 clk_req_support = false;
1777                                 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
1778                                 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
1779                                         clk_req_support = true;
1780                         } else {
1781                                 clk_req_support = false;
1782                         }
1783
1784                         if (clk_req_support) {
1785                                 orig = data = RREG32_PCIE(ixPCIE_LC_CNTL2);
1786                                 data |= PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L1_MASK |
1787                                         PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L23_MASK;
1788                                 if (orig != data)
1789                                         WREG32_PCIE(ixPCIE_LC_CNTL2, data);
1790
1791                                 orig = data = RREG32_SMC(ixTHM_CLK_CNTL);
1792                                 data &= ~(THM_CLK_CNTL__CMON_CLK_SEL_MASK |
1793                                         THM_CLK_CNTL__TMON_CLK_SEL_MASK);
1794                                 data |= (1 << THM_CLK_CNTL__CMON_CLK_SEL__SHIFT) |
1795                                         (1 << THM_CLK_CNTL__TMON_CLK_SEL__SHIFT);
1796                                 if (orig != data)
1797                                         WREG32_SMC(ixTHM_CLK_CNTL, data);
1798
1799                                 orig = data = RREG32_SMC(ixMISC_CLK_CTRL);
1800                                 data &= ~(MISC_CLK_CTRL__DEEP_SLEEP_CLK_SEL_MASK |
1801                                         MISC_CLK_CTRL__ZCLK_SEL_MASK);
1802                                 data |= (1 << MISC_CLK_CTRL__DEEP_SLEEP_CLK_SEL__SHIFT) |
1803                                         (1 << MISC_CLK_CTRL__ZCLK_SEL__SHIFT);
1804                                 if (orig != data)
1805                                         WREG32_SMC(ixMISC_CLK_CTRL, data);
1806
1807                                 orig = data = RREG32_SMC(ixCG_CLKPIN_CNTL);
1808                                 data &= ~CG_CLKPIN_CNTL__BCLK_AS_XCLK_MASK;
1809                                 if (orig != data)
1810                                         WREG32_SMC(ixCG_CLKPIN_CNTL, data);
1811
1812                                 orig = data = RREG32_SMC(ixCG_CLKPIN_CNTL_2);
1813                                 data &= ~CG_CLKPIN_CNTL_2__FORCE_BIF_REFCLK_EN_MASK;
1814                                 if (orig != data)
1815                                         WREG32_SMC(ixCG_CLKPIN_CNTL_2, data);
1816
1817                                 orig = data = RREG32_SMC(ixMPLL_BYPASSCLK_SEL);
1818                                 data &= ~MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL_MASK;
1819                                 data |= (4 << MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL__SHIFT);
1820                                 if (orig != data)
1821                                         WREG32_SMC(ixMPLL_BYPASSCLK_SEL, data);
1822                         }
1823                 }
1824         } else {
1825                 if (orig != data)
1826                         WREG32_PCIE(ixPCIE_LC_CNTL, data);
1827         }
1828
1829         orig = data = RREG32_PCIE(ixPCIE_CNTL2);
1830         data |= PCIE_CNTL2__SLV_MEM_LS_EN_MASK |
1831                 PCIE_CNTL2__MST_MEM_LS_EN_MASK |
1832                 PCIE_CNTL2__REPLAY_MEM_LS_EN_MASK;
1833         if (orig != data)
1834                 WREG32_PCIE(ixPCIE_CNTL2, data);
1835
1836         if (!disable_l0s) {
1837                 data = RREG32_PCIE(ixPCIE_LC_N_FTS_CNTL);
1838                 if ((data & PCIE_LC_N_FTS_CNTL__LC_N_FTS_MASK) ==
1839                                 PCIE_LC_N_FTS_CNTL__LC_N_FTS_MASK) {
1840                         data = RREG32_PCIE(ixPCIE_LC_STATUS1);
1841                         if ((data & PCIE_LC_STATUS1__LC_REVERSE_XMIT_MASK) &&
1842                         (data & PCIE_LC_STATUS1__LC_REVERSE_RCVR_MASK)) {
1843                                 orig = data = RREG32_PCIE(ixPCIE_LC_CNTL);
1844                                 data &= ~PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK;
1845                                 if (orig != data)
1846                                         WREG32_PCIE(ixPCIE_LC_CNTL, data);
1847                         }
1848                 }
1849         }
1850 }
1851
1852 static uint32_t cik_get_rev_id(struct amdgpu_device *adev)
1853 {
1854         return (RREG32(mmCC_DRM_ID_STRAPS) & CC_DRM_ID_STRAPS__ATI_REV_ID_MASK)
1855                 >> CC_DRM_ID_STRAPS__ATI_REV_ID__SHIFT;
1856 }
1857
1858 static void cik_flush_hdp(struct amdgpu_device *adev, struct amdgpu_ring *ring)
1859 {
1860         if (!ring || !ring->funcs->emit_wreg) {
1861                 WREG32(mmHDP_MEM_COHERENCY_FLUSH_CNTL, 1);
1862                 RREG32(mmHDP_MEM_COHERENCY_FLUSH_CNTL);
1863         } else {
1864                 amdgpu_ring_emit_wreg(ring, mmHDP_MEM_COHERENCY_FLUSH_CNTL, 1);
1865         }
1866 }
1867
1868 static void cik_invalidate_hdp(struct amdgpu_device *adev,
1869                                struct amdgpu_ring *ring)
1870 {
1871         if (!ring || !ring->funcs->emit_wreg) {
1872                 WREG32(mmHDP_DEBUG0, 1);
1873                 RREG32(mmHDP_DEBUG0);
1874         } else {
1875                 amdgpu_ring_emit_wreg(ring, mmHDP_DEBUG0, 1);
1876         }
1877 }
1878
1879 static bool cik_need_full_reset(struct amdgpu_device *adev)
1880 {
1881         /* change this when we support soft reset */
1882         return true;
1883 }
1884
1885 static void cik_get_pcie_usage(struct amdgpu_device *adev, uint64_t *count0,
1886                                uint64_t *count1)
1887 {
1888         uint32_t perfctr = 0;
1889         uint64_t cnt0_of, cnt1_of;
1890         int tmp;
1891
1892         /* This reports 0 on APUs, so return to avoid writing/reading registers
1893          * that may or may not be different from their GPU counterparts
1894          */
1895         if (adev->flags & AMD_IS_APU)
1896                 return;
1897
1898         /* Set the 2 events that we wish to watch, defined above */
1899         /* Reg 40 is # received msgs, Reg 104 is # of posted requests sent */
1900         perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK, EVENT0_SEL, 40);
1901         perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK, EVENT1_SEL, 104);
1902
1903         /* Write to enable desired perf counters */
1904         WREG32_PCIE(ixPCIE_PERF_CNTL_TXCLK, perfctr);
1905         /* Zero out and enable the perf counters
1906          * Write 0x5:
1907          * Bit 0 = Start all counters(1)
1908          * Bit 2 = Global counter reset enable(1)
1909          */
1910         WREG32_PCIE(ixPCIE_PERF_COUNT_CNTL, 0x00000005);
1911
1912         msleep(1000);
1913
1914         /* Load the shadow and disable the perf counters
1915          * Write 0x2:
1916          * Bit 0 = Stop counters(0)
1917          * Bit 1 = Load the shadow counters(1)
1918          */
1919         WREG32_PCIE(ixPCIE_PERF_COUNT_CNTL, 0x00000002);
1920
1921         /* Read register values to get any >32bit overflow */
1922         tmp = RREG32_PCIE(ixPCIE_PERF_CNTL_TXCLK);
1923         cnt0_of = REG_GET_FIELD(tmp, PCIE_PERF_CNTL_TXCLK, COUNTER0_UPPER);
1924         cnt1_of = REG_GET_FIELD(tmp, PCIE_PERF_CNTL_TXCLK, COUNTER1_UPPER);
1925
1926         /* Get the values and add the overflow */
1927         *count0 = RREG32_PCIE(ixPCIE_PERF_COUNT0_TXCLK) | (cnt0_of << 32);
1928         *count1 = RREG32_PCIE(ixPCIE_PERF_COUNT1_TXCLK) | (cnt1_of << 32);
1929 }
1930
1931 static bool cik_need_reset_on_init(struct amdgpu_device *adev)
1932 {
1933         u32 clock_cntl, pc;
1934
1935         if (adev->flags & AMD_IS_APU)
1936                 return false;
1937
1938         /* check if the SMC is already running */
1939         clock_cntl = RREG32_SMC(ixSMC_SYSCON_CLOCK_CNTL_0);
1940         pc = RREG32_SMC(ixSMC_PC_C);
1941         if ((0 == REG_GET_FIELD(clock_cntl, SMC_SYSCON_CLOCK_CNTL_0, ck_disable)) &&
1942             (0x20100 <= pc))
1943                 return true;
1944
1945         return false;
1946 }
1947
1948 static uint64_t cik_get_pcie_replay_count(struct amdgpu_device *adev)
1949 {
1950         uint64_t nak_r, nak_g;
1951
1952         /* Get the number of NAKs received and generated */
1953         nak_r = RREG32_PCIE(ixPCIE_RX_NUM_NAK);
1954         nak_g = RREG32_PCIE(ixPCIE_RX_NUM_NAK_GENERATED);
1955
1956         /* Add the total number of NAKs, i.e the number of replays */
1957         return (nak_r + nak_g);
1958 }
1959
1960 static void cik_pre_asic_init(struct amdgpu_device *adev)
1961 {
1962 }
1963
1964 static const struct amdgpu_asic_funcs cik_asic_funcs =
1965 {
1966         .read_disabled_bios = &cik_read_disabled_bios,
1967         .read_bios_from_rom = &cik_read_bios_from_rom,
1968         .read_register = &cik_read_register,
1969         .reset = &cik_asic_reset,
1970         .reset_method = &cik_asic_reset_method,
1971         .set_vga_state = &cik_vga_set_state,
1972         .get_xclk = &cik_get_xclk,
1973         .set_uvd_clocks = &cik_set_uvd_clocks,
1974         .set_vce_clocks = &cik_set_vce_clocks,
1975         .get_config_memsize = &cik_get_config_memsize,
1976         .flush_hdp = &cik_flush_hdp,
1977         .invalidate_hdp = &cik_invalidate_hdp,
1978         .need_full_reset = &cik_need_full_reset,
1979         .init_doorbell_index = &legacy_doorbell_index_init,
1980         .get_pcie_usage = &cik_get_pcie_usage,
1981         .need_reset_on_init = &cik_need_reset_on_init,
1982         .get_pcie_replay_count = &cik_get_pcie_replay_count,
1983         .supports_baco = &cik_asic_supports_baco,
1984         .pre_asic_init = &cik_pre_asic_init,
1985         .query_video_codecs = &cik_query_video_codecs,
1986 };
1987
1988 static int cik_common_early_init(struct amdgpu_ip_block *ip_block)
1989 {
1990         struct amdgpu_device *adev = ip_block->adev;
1991
1992         adev->smc_rreg = &cik_smc_rreg;
1993         adev->smc_wreg = &cik_smc_wreg;
1994         adev->pcie_rreg = &cik_pcie_rreg;
1995         adev->pcie_wreg = &cik_pcie_wreg;
1996         adev->uvd_ctx_rreg = &cik_uvd_ctx_rreg;
1997         adev->uvd_ctx_wreg = &cik_uvd_ctx_wreg;
1998         adev->didt_rreg = &cik_didt_rreg;
1999         adev->didt_wreg = &cik_didt_wreg;
2000
2001         adev->asic_funcs = &cik_asic_funcs;
2002
2003         adev->rev_id = cik_get_rev_id(adev);
2004         adev->external_rev_id = 0xFF;
2005         switch (adev->asic_type) {
2006         case CHIP_BONAIRE:
2007                 adev->cg_flags =
2008                         AMD_CG_SUPPORT_GFX_MGCG |
2009                         AMD_CG_SUPPORT_GFX_MGLS |
2010                         /*AMD_CG_SUPPORT_GFX_CGCG |*/
2011                         AMD_CG_SUPPORT_GFX_CGLS |
2012                         AMD_CG_SUPPORT_GFX_CGTS |
2013                         AMD_CG_SUPPORT_GFX_CGTS_LS |
2014                         AMD_CG_SUPPORT_GFX_CP_LS |
2015                         AMD_CG_SUPPORT_MC_LS |
2016                         AMD_CG_SUPPORT_MC_MGCG |
2017                         AMD_CG_SUPPORT_SDMA_MGCG |
2018                         AMD_CG_SUPPORT_SDMA_LS |
2019                         AMD_CG_SUPPORT_BIF_LS |
2020                         AMD_CG_SUPPORT_VCE_MGCG |
2021                         AMD_CG_SUPPORT_UVD_MGCG |
2022                         AMD_CG_SUPPORT_HDP_LS |
2023                         AMD_CG_SUPPORT_HDP_MGCG;
2024                 adev->pg_flags = 0;
2025                 adev->external_rev_id = adev->rev_id + 0x14;
2026                 break;
2027         case CHIP_HAWAII:
2028                 adev->cg_flags =
2029                         AMD_CG_SUPPORT_GFX_MGCG |
2030                         AMD_CG_SUPPORT_GFX_MGLS |
2031                         /*AMD_CG_SUPPORT_GFX_CGCG |*/
2032                         AMD_CG_SUPPORT_GFX_CGLS |
2033                         AMD_CG_SUPPORT_GFX_CGTS |
2034                         AMD_CG_SUPPORT_GFX_CP_LS |
2035                         AMD_CG_SUPPORT_MC_LS |
2036                         AMD_CG_SUPPORT_MC_MGCG |
2037                         AMD_CG_SUPPORT_SDMA_MGCG |
2038                         AMD_CG_SUPPORT_SDMA_LS |
2039                         AMD_CG_SUPPORT_BIF_LS |
2040                         AMD_CG_SUPPORT_VCE_MGCG |
2041                         AMD_CG_SUPPORT_UVD_MGCG |
2042                         AMD_CG_SUPPORT_HDP_LS |
2043                         AMD_CG_SUPPORT_HDP_MGCG;
2044                 adev->pg_flags = 0;
2045                 adev->external_rev_id = 0x28;
2046                 break;
2047         case CHIP_KAVERI:
2048                 adev->cg_flags =
2049                         AMD_CG_SUPPORT_GFX_MGCG |
2050                         AMD_CG_SUPPORT_GFX_MGLS |
2051                         /*AMD_CG_SUPPORT_GFX_CGCG |*/
2052                         AMD_CG_SUPPORT_GFX_CGLS |
2053                         AMD_CG_SUPPORT_GFX_CGTS |
2054                         AMD_CG_SUPPORT_GFX_CGTS_LS |
2055                         AMD_CG_SUPPORT_GFX_CP_LS |
2056                         AMD_CG_SUPPORT_SDMA_MGCG |
2057                         AMD_CG_SUPPORT_SDMA_LS |
2058                         AMD_CG_SUPPORT_BIF_LS |
2059                         AMD_CG_SUPPORT_VCE_MGCG |
2060                         AMD_CG_SUPPORT_UVD_MGCG |
2061                         AMD_CG_SUPPORT_HDP_LS |
2062                         AMD_CG_SUPPORT_HDP_MGCG;
2063                 adev->pg_flags =
2064                         /*AMD_PG_SUPPORT_GFX_PG |
2065                           AMD_PG_SUPPORT_GFX_SMG |
2066                           AMD_PG_SUPPORT_GFX_DMG |*/
2067                         AMD_PG_SUPPORT_UVD |
2068                         AMD_PG_SUPPORT_VCE |
2069                         /*  AMD_PG_SUPPORT_CP |
2070                           AMD_PG_SUPPORT_GDS |
2071                           AMD_PG_SUPPORT_RLC_SMU_HS |
2072                           AMD_PG_SUPPORT_ACP |
2073                           AMD_PG_SUPPORT_SAMU |*/
2074                         0;
2075                 if (adev->pdev->device == 0x1312 ||
2076                         adev->pdev->device == 0x1316 ||
2077                         adev->pdev->device == 0x1317)
2078                         adev->external_rev_id = 0x41;
2079                 else
2080                         adev->external_rev_id = 0x1;
2081                 break;
2082         case CHIP_KABINI:
2083         case CHIP_MULLINS:
2084                 adev->cg_flags =
2085                         AMD_CG_SUPPORT_GFX_MGCG |
2086                         AMD_CG_SUPPORT_GFX_MGLS |
2087                         /*AMD_CG_SUPPORT_GFX_CGCG |*/
2088                         AMD_CG_SUPPORT_GFX_CGLS |
2089                         AMD_CG_SUPPORT_GFX_CGTS |
2090                         AMD_CG_SUPPORT_GFX_CGTS_LS |
2091                         AMD_CG_SUPPORT_GFX_CP_LS |
2092                         AMD_CG_SUPPORT_SDMA_MGCG |
2093                         AMD_CG_SUPPORT_SDMA_LS |
2094                         AMD_CG_SUPPORT_BIF_LS |
2095                         AMD_CG_SUPPORT_VCE_MGCG |
2096                         AMD_CG_SUPPORT_UVD_MGCG |
2097                         AMD_CG_SUPPORT_HDP_LS |
2098                         AMD_CG_SUPPORT_HDP_MGCG;
2099                 adev->pg_flags =
2100                         /*AMD_PG_SUPPORT_GFX_PG |
2101                           AMD_PG_SUPPORT_GFX_SMG | */
2102                         AMD_PG_SUPPORT_UVD |
2103                         /*AMD_PG_SUPPORT_VCE |
2104                           AMD_PG_SUPPORT_CP |
2105                           AMD_PG_SUPPORT_GDS |
2106                           AMD_PG_SUPPORT_RLC_SMU_HS |
2107                           AMD_PG_SUPPORT_SAMU |*/
2108                         0;
2109                 if (adev->asic_type == CHIP_KABINI) {
2110                         if (adev->rev_id == 0)
2111                                 adev->external_rev_id = 0x81;
2112                         else if (adev->rev_id == 1)
2113                                 adev->external_rev_id = 0x82;
2114                         else if (adev->rev_id == 2)
2115                                 adev->external_rev_id = 0x85;
2116                 } else
2117                         adev->external_rev_id = adev->rev_id + 0xa1;
2118                 break;
2119         default:
2120                 /* FIXME: not supported yet */
2121                 return -EINVAL;
2122         }
2123
2124         return 0;
2125 }
2126
2127 static int cik_common_hw_init(struct amdgpu_ip_block *ip_block)
2128 {
2129         struct amdgpu_device *adev = ip_block->adev;
2130
2131         /* move the golden regs per IP block */
2132         cik_init_golden_registers(adev);
2133         /* enable pcie gen2/3 link */
2134         cik_pcie_gen3_enable(adev);
2135         /* enable aspm */
2136         cik_program_aspm(adev);
2137
2138         return 0;
2139 }
2140
2141 static int cik_common_hw_fini(struct amdgpu_ip_block *ip_block)
2142 {
2143         return 0;
2144 }
2145
2146 static int cik_common_resume(struct amdgpu_ip_block *ip_block)
2147 {
2148         return cik_common_hw_init(ip_block);
2149 }
2150
2151 static bool cik_common_is_idle(void *handle)
2152 {
2153         return true;
2154 }
2155
2156
2157
2158 static int cik_common_soft_reset(struct amdgpu_ip_block *ip_block)
2159 {
2160         /* XXX hard reset?? */
2161         return 0;
2162 }
2163
2164 static int cik_common_set_clockgating_state(struct amdgpu_ip_block *ip_block,
2165                                             enum amd_clockgating_state state)
2166 {
2167         return 0;
2168 }
2169
2170 static int cik_common_set_powergating_state(struct amdgpu_ip_block *ip_block,
2171                                             enum amd_powergating_state state)
2172 {
2173         return 0;
2174 }
2175
2176 static const struct amd_ip_funcs cik_common_ip_funcs = {
2177         .name = "cik_common",
2178         .early_init = cik_common_early_init,
2179         .hw_init = cik_common_hw_init,
2180         .hw_fini = cik_common_hw_fini,
2181         .resume = cik_common_resume,
2182         .is_idle = cik_common_is_idle,
2183         .soft_reset = cik_common_soft_reset,
2184         .set_clockgating_state = cik_common_set_clockgating_state,
2185         .set_powergating_state = cik_common_set_powergating_state,
2186 };
2187
2188 static const struct amdgpu_ip_block_version cik_common_ip_block =
2189 {
2190         .type = AMD_IP_BLOCK_TYPE_COMMON,
2191         .major = 1,
2192         .minor = 0,
2193         .rev = 0,
2194         .funcs = &cik_common_ip_funcs,
2195 };
2196
2197 int cik_set_ip_blocks(struct amdgpu_device *adev)
2198 {
2199         switch (adev->asic_type) {
2200         case CHIP_BONAIRE:
2201                 amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2202                 amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2203                 amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2204                 amdgpu_device_ip_block_add(adev, &gfx_v7_2_ip_block);
2205                 amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2206                 amdgpu_device_ip_block_add(adev, &pp_smu_ip_block);
2207                 if (adev->enable_virtual_display)
2208                         amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block);
2209 #if defined(CONFIG_DRM_AMD_DC)
2210                 else if (amdgpu_device_has_dc_support(adev))
2211                         amdgpu_device_ip_block_add(adev, &dm_ip_block);
2212 #endif
2213                 else
2214                         amdgpu_device_ip_block_add(adev, &dce_v8_2_ip_block);
2215                 amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2216                 amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
2217                 break;
2218         case CHIP_HAWAII:
2219                 amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2220                 amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2221                 amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2222                 amdgpu_device_ip_block_add(adev, &gfx_v7_3_ip_block);
2223                 amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2224                 amdgpu_device_ip_block_add(adev, &pp_smu_ip_block);
2225                 if (adev->enable_virtual_display)
2226                         amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block);
2227 #if defined(CONFIG_DRM_AMD_DC)
2228                 else if (amdgpu_device_has_dc_support(adev))
2229                         amdgpu_device_ip_block_add(adev, &dm_ip_block);
2230 #endif
2231                 else
2232                         amdgpu_device_ip_block_add(adev, &dce_v8_5_ip_block);
2233                 amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2234                 amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
2235                 break;
2236         case CHIP_KAVERI:
2237                 amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2238                 amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2239                 amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2240                 amdgpu_device_ip_block_add(adev, &gfx_v7_1_ip_block);
2241                 amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2242                 amdgpu_device_ip_block_add(adev, &kv_smu_ip_block);
2243                 if (adev->enable_virtual_display)
2244                         amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block);
2245 #if defined(CONFIG_DRM_AMD_DC)
2246                 else if (amdgpu_device_has_dc_support(adev))
2247                         amdgpu_device_ip_block_add(adev, &dm_ip_block);
2248 #endif
2249                 else
2250                         amdgpu_device_ip_block_add(adev, &dce_v8_1_ip_block);
2251
2252                 amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2253                 amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
2254                 break;
2255         case CHIP_KABINI:
2256         case CHIP_MULLINS:
2257                 amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2258                 amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2259                 amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2260                 amdgpu_device_ip_block_add(adev, &gfx_v7_2_ip_block);
2261                 amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2262                 amdgpu_device_ip_block_add(adev, &kv_smu_ip_block);
2263                 if (adev->enable_virtual_display)
2264                         amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block);
2265 #if defined(CONFIG_DRM_AMD_DC)
2266                 else if (amdgpu_device_has_dc_support(adev))
2267                         amdgpu_device_ip_block_add(adev, &dm_ip_block);
2268 #endif
2269                 else
2270                         amdgpu_device_ip_block_add(adev, &dce_v8_3_ip_block);
2271                 amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2272                 amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
2273                 break;
2274         default:
2275                 /* FIXME: not supported yet */
2276                 return -EINVAL;
2277         }
2278         return 0;
2279 }