drivers/gpu/drm/radeon/evergreen.c

   1 /*
   2  * Copyright 2010 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Alex Deucher
  23  */
  24
  25 #include <linux/firmware.h>
  26 #include <linux/pci.h>
  27 #include <linux/slab.h>
  28
  29 #include <drm/drm_edid.h>
  30 #include <drm/drm_vblank.h>
  31 #include <drm/radeon_drm.h>
  32 #include <drm/drm_fourcc.h>
  33 #include <drm/drm_framebuffer.h>
  34
  35 #include "atom.h"
  36 #include "avivod.h"
  37 #include "cik.h"
  38 #include "ni.h"
  39 #include "rv770.h"
  40 #include "evergreen.h"
  41 #include "evergreen_blit_shaders.h"
  42 #include "evergreen_reg.h"
  43 #include "evergreend.h"
  44 #include "radeon.h"
  45 #include "radeon_asic.h"
  46 #include "radeon_audio.h"
  47 #include "radeon_ucode.h"
  48 #include "si.h"
  49
  50 #define DC_HPDx_CONTROL(x)        (DC_HPD1_CONTROL     + (x * 0xc))
  51 #define DC_HPDx_INT_CONTROL(x)    (DC_HPD1_INT_CONTROL + (x * 0xc))
  52 #define DC_HPDx_INT_STATUS_REG(x) (DC_HPD1_INT_STATUS  + (x * 0xc))
  53
  54 /*
  55  * Indirect registers accessor
  56  */
  57 u32 eg_cg_rreg(struct radeon_device *rdev, u32 reg)
  58 {
  59         unsigned long flags;
  60         u32 r;
  61
  62         spin_lock_irqsave(&rdev->cg_idx_lock, flags);
  63         WREG32(EVERGREEN_CG_IND_ADDR, ((reg) & 0xffff));
  64         r = RREG32(EVERGREEN_CG_IND_DATA);
  65         spin_unlock_irqrestore(&rdev->cg_idx_lock, flags);
  66         return r;
  67 }
  68
  69 void eg_cg_wreg(struct radeon_device *rdev, u32 reg, u32 v)
  70 {
  71         unsigned long flags;
  72
  73         spin_lock_irqsave(&rdev->cg_idx_lock, flags);
  74         WREG32(EVERGREEN_CG_IND_ADDR, ((reg) & 0xffff));
  75         WREG32(EVERGREEN_CG_IND_DATA, (v));
  76         spin_unlock_irqrestore(&rdev->cg_idx_lock, flags);
  77 }
  78
  79 u32 eg_pif_phy0_rreg(struct radeon_device *rdev, u32 reg)
  80 {
  81         unsigned long flags;
  82         u32 r;
  83
  84         spin_lock_irqsave(&rdev->pif_idx_lock, flags);
  85         WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
  86         r = RREG32(EVERGREEN_PIF_PHY0_DATA);
  87         spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
  88         return r;
  89 }
  90
  91 void eg_pif_phy0_wreg(struct radeon_device *rdev, u32 reg, u32 v)
  92 {
  93         unsigned long flags;
  94
  95         spin_lock_irqsave(&rdev->pif_idx_lock, flags);
  96         WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
  97         WREG32(EVERGREEN_PIF_PHY0_DATA, (v));
  98         spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
  99 }
 100
 101 u32 eg_pif_phy1_rreg(struct radeon_device *rdev, u32 reg)
 102 {
 103         unsigned long flags;
 104         u32 r;
 105
 106         spin_lock_irqsave(&rdev->pif_idx_lock, flags);
 107         WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
 108         r = RREG32(EVERGREEN_PIF_PHY1_DATA);
 109         spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
 110         return r;
 111 }
 112
 113 void eg_pif_phy1_wreg(struct radeon_device *rdev, u32 reg, u32 v)
 114 {
 115         unsigned long flags;
 116
 117         spin_lock_irqsave(&rdev->pif_idx_lock, flags);
 118         WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
 119         WREG32(EVERGREEN_PIF_PHY1_DATA, (v));
 120         spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
 121 }
 122
 123 static const u32 crtc_offsets[6] =
 124 {
 125         EVERGREEN_CRTC0_REGISTER_OFFSET,
 126         EVERGREEN_CRTC1_REGISTER_OFFSET,
 127         EVERGREEN_CRTC2_REGISTER_OFFSET,
 128         EVERGREEN_CRTC3_REGISTER_OFFSET,
 129         EVERGREEN_CRTC4_REGISTER_OFFSET,
 130         EVERGREEN_CRTC5_REGISTER_OFFSET
 131 };
 132
 133 #include "clearstate_evergreen.h"
 134
 135 static const u32 sumo_rlc_save_restore_register_list[] =
 136 {
 137         0x98fc,
 138         0x9830,
 139         0x9834,
 140         0x9838,
 141         0x9870,
 142         0x9874,
 143         0x8a14,
 144         0x8b24,
 145         0x8bcc,
 146         0x8b10,
 147         0x8d00,
 148         0x8d04,
 149         0x8c00,
 150         0x8c04,
 151         0x8c08,
 152         0x8c0c,
 153         0x8d8c,
 154         0x8c20,
 155         0x8c24,
 156         0x8c28,
 157         0x8c18,
 158         0x8c1c,
 159         0x8cf0,
 160         0x8e2c,
 161         0x8e38,
 162         0x8c30,
 163         0x9508,
 164         0x9688,
 165         0x9608,
 166         0x960c,
 167         0x9610,
 168         0x9614,
 169         0x88c4,
 170         0x88d4,
 171         0xa008,
 172         0x900c,
 173         0x9100,
 174         0x913c,
 175         0x98f8,
 176         0x98f4,
 177         0x9b7c,
 178         0x3f8c,
 179         0x8950,
 180         0x8954,
 181         0x8a18,
 182         0x8b28,
 183         0x9144,
 184         0x9148,
 185         0x914c,
 186         0x3f90,
 187         0x3f94,
 188         0x915c,
 189         0x9160,
 190         0x9178,
 191         0x917c,
 192         0x9180,
 193         0x918c,
 194         0x9190,
 195         0x9194,
 196         0x9198,
 197         0x919c,
 198         0x91a8,
 199         0x91ac,
 200         0x91b0,
 201         0x91b4,
 202         0x91b8,
 203         0x91c4,
 204         0x91c8,
 205         0x91cc,
 206         0x91d0,
 207         0x91d4,
 208         0x91e0,
 209         0x91e4,
 210         0x91ec,
 211         0x91f0,
 212         0x91f4,
 213         0x9200,
 214         0x9204,
 215         0x929c,
 216         0x9150,
 217         0x802c,
 218 };
 219
 220 static void evergreen_gpu_init(struct radeon_device *rdev);
 221 void evergreen_fini(struct radeon_device *rdev);
 222 void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
 223 void evergreen_program_aspm(struct radeon_device *rdev);
 224
 225 static const u32 evergreen_golden_registers[] =
 226 {
 227         0x3f90, 0xffff0000, 0xff000000,
 228         0x9148, 0xffff0000, 0xff000000,
 229         0x3f94, 0xffff0000, 0xff000000,
 230         0x914c, 0xffff0000, 0xff000000,
 231         0x9b7c, 0xffffffff, 0x00000000,
 232         0x8a14, 0xffffffff, 0x00000007,
 233         0x8b10, 0xffffffff, 0x00000000,
 234         0x960c, 0xffffffff, 0x54763210,
 235         0x88c4, 0xffffffff, 0x000000c2,
 236         0x88d4, 0xffffffff, 0x00000010,
 237         0x8974, 0xffffffff, 0x00000000,
 238         0xc78, 0x00000080, 0x00000080,
 239         0x5eb4, 0xffffffff, 0x00000002,
 240         0x5e78, 0xffffffff, 0x001000f0,
 241         0x6104, 0x01000300, 0x00000000,
 242         0x5bc0, 0x00300000, 0x00000000,
 243         0x7030, 0xffffffff, 0x00000011,
 244         0x7c30, 0xffffffff, 0x00000011,
 245         0x10830, 0xffffffff, 0x00000011,
 246         0x11430, 0xffffffff, 0x00000011,
 247         0x12030, 0xffffffff, 0x00000011,
 248         0x12c30, 0xffffffff, 0x00000011,
 249         0xd02c, 0xffffffff, 0x08421000,
 250         0x240c, 0xffffffff, 0x00000380,
 251         0x8b24, 0xffffffff, 0x00ff0fff,
 252         0x28a4c, 0x06000000, 0x06000000,
 253         0x10c, 0x00000001, 0x00000001,
 254         0x8d00, 0xffffffff, 0x100e4848,
 255         0x8d04, 0xffffffff, 0x00164745,
 256         0x8c00, 0xffffffff, 0xe4000003,
 257         0x8c04, 0xffffffff, 0x40600060,
 258         0x8c08, 0xffffffff, 0x001c001c,
 259         0x8cf0, 0xffffffff, 0x08e00620,
 260         0x8c20, 0xffffffff, 0x00800080,
 261         0x8c24, 0xffffffff, 0x00800080,
 262         0x8c18, 0xffffffff, 0x20202078,
 263         0x8c1c, 0xffffffff, 0x00001010,
 264         0x28350, 0xffffffff, 0x00000000,
 265         0xa008, 0xffffffff, 0x00010000,
 266         0x5c4, 0xffffffff, 0x00000001,
 267         0x9508, 0xffffffff, 0x00000002,
 268         0x913c, 0x0000000f, 0x0000000a
 269 };
 270
 271 static const u32 evergreen_golden_registers2[] =
 272 {
 273         0x2f4c, 0xffffffff, 0x00000000,
 274         0x54f4, 0xffffffff, 0x00000000,
 275         0x54f0, 0xffffffff, 0x00000000,
 276         0x5498, 0xffffffff, 0x00000000,
 277         0x549c, 0xffffffff, 0x00000000,
 278         0x5494, 0xffffffff, 0x00000000,
 279         0x53cc, 0xffffffff, 0x00000000,
 280         0x53c8, 0xffffffff, 0x00000000,
 281         0x53c4, 0xffffffff, 0x00000000,
 282         0x53c0, 0xffffffff, 0x00000000,
 283         0x53bc, 0xffffffff, 0x00000000,
 284         0x53b8, 0xffffffff, 0x00000000,
 285         0x53b4, 0xffffffff, 0x00000000,
 286         0x53b0, 0xffffffff, 0x00000000
 287 };
 288
 289 static const u32 cypress_mgcg_init[] =
 290 {
 291         0x802c, 0xffffffff, 0xc0000000,
 292         0x5448, 0xffffffff, 0x00000100,
 293         0x55e4, 0xffffffff, 0x00000100,
 294         0x160c, 0xffffffff, 0x00000100,
 295         0x5644, 0xffffffff, 0x00000100,
 296         0xc164, 0xffffffff, 0x00000100,
 297         0x8a18, 0xffffffff, 0x00000100,
 298         0x897c, 0xffffffff, 0x06000100,
 299         0x8b28, 0xffffffff, 0x00000100,
 300         0x9144, 0xffffffff, 0x00000100,
 301         0x9a60, 0xffffffff, 0x00000100,
 302         0x9868, 0xffffffff, 0x00000100,
 303         0x8d58, 0xffffffff, 0x00000100,
 304         0x9510, 0xffffffff, 0x00000100,
 305         0x949c, 0xffffffff, 0x00000100,
 306         0x9654, 0xffffffff, 0x00000100,
 307         0x9030, 0xffffffff, 0x00000100,
 308         0x9034, 0xffffffff, 0x00000100,
 309         0x9038, 0xffffffff, 0x00000100,
 310         0x903c, 0xffffffff, 0x00000100,
 311         0x9040, 0xffffffff, 0x00000100,
 312         0xa200, 0xffffffff, 0x00000100,
 313         0xa204, 0xffffffff, 0x00000100,
 314         0xa208, 0xffffffff, 0x00000100,
 315         0xa20c, 0xffffffff, 0x00000100,
 316         0x971c, 0xffffffff, 0x00000100,
 317         0x977c, 0xffffffff, 0x00000100,
 318         0x3f80, 0xffffffff, 0x00000100,
 319         0xa210, 0xffffffff, 0x00000100,
 320         0xa214, 0xffffffff, 0x00000100,
 321         0x4d8, 0xffffffff, 0x00000100,
 322         0x9784, 0xffffffff, 0x00000100,
 323         0x9698, 0xffffffff, 0x00000100,
 324         0x4d4, 0xffffffff, 0x00000200,
 325         0x30cc, 0xffffffff, 0x00000100,
 326         0xd0c0, 0xffffffff, 0xff000100,
 327         0x802c, 0xffffffff, 0x40000000,
 328         0x915c, 0xffffffff, 0x00010000,
 329         0x9160, 0xffffffff, 0x00030002,
 330         0x9178, 0xffffffff, 0x00070000,
 331         0x917c, 0xffffffff, 0x00030002,
 332         0x9180, 0xffffffff, 0x00050004,
 333         0x918c, 0xffffffff, 0x00010006,
 334         0x9190, 0xffffffff, 0x00090008,
 335         0x9194, 0xffffffff, 0x00070000,
 336         0x9198, 0xffffffff, 0x00030002,
 337         0x919c, 0xffffffff, 0x00050004,
 338         0x91a8, 0xffffffff, 0x00010006,
 339         0x91ac, 0xffffffff, 0x00090008,
 340         0x91b0, 0xffffffff, 0x00070000,
 341         0x91b4, 0xffffffff, 0x00030002,
 342         0x91b8, 0xffffffff, 0x00050004,
 343         0x91c4, 0xffffffff, 0x00010006,
 344         0x91c8, 0xffffffff, 0x00090008,
 345         0x91cc, 0xffffffff, 0x00070000,
 346         0x91d0, 0xffffffff, 0x00030002,
 347         0x91d4, 0xffffffff, 0x00050004,
 348         0x91e0, 0xffffffff, 0x00010006,
 349         0x91e4, 0xffffffff, 0x00090008,
 350         0x91e8, 0xffffffff, 0x00000000,
 351         0x91ec, 0xffffffff, 0x00070000,
 352         0x91f0, 0xffffffff, 0x00030002,
 353         0x91f4, 0xffffffff, 0x00050004,
 354         0x9200, 0xffffffff, 0x00010006,
 355         0x9204, 0xffffffff, 0x00090008,
 356         0x9208, 0xffffffff, 0x00070000,
 357         0x920c, 0xffffffff, 0x00030002,
 358         0x9210, 0xffffffff, 0x00050004,
 359         0x921c, 0xffffffff, 0x00010006,
 360         0x9220, 0xffffffff, 0x00090008,
 361         0x9224, 0xffffffff, 0x00070000,
 362         0x9228, 0xffffffff, 0x00030002,
 363         0x922c, 0xffffffff, 0x00050004,
 364         0x9238, 0xffffffff, 0x00010006,
 365         0x923c, 0xffffffff, 0x00090008,
 366         0x9240, 0xffffffff, 0x00070000,
 367         0x9244, 0xffffffff, 0x00030002,
 368         0x9248, 0xffffffff, 0x00050004,
 369         0x9254, 0xffffffff, 0x00010006,
 370         0x9258, 0xffffffff, 0x00090008,
 371         0x925c, 0xffffffff, 0x00070000,
 372         0x9260, 0xffffffff, 0x00030002,
 373         0x9264, 0xffffffff, 0x00050004,
 374         0x9270, 0xffffffff, 0x00010006,
 375         0x9274, 0xffffffff, 0x00090008,
 376         0x9278, 0xffffffff, 0x00070000,
 377         0x927c, 0xffffffff, 0x00030002,
 378         0x9280, 0xffffffff, 0x00050004,
 379         0x928c, 0xffffffff, 0x00010006,
 380         0x9290, 0xffffffff, 0x00090008,
 381         0x9294, 0xffffffff, 0x00000000,
 382         0x929c, 0xffffffff, 0x00000001,
 383         0x802c, 0xffffffff, 0x40010000,
 384         0x915c, 0xffffffff, 0x00010000,
 385         0x9160, 0xffffffff, 0x00030002,
 386         0x9178, 0xffffffff, 0x00070000,
 387         0x917c, 0xffffffff, 0x00030002,
 388         0x9180, 0xffffffff, 0x00050004,
 389         0x918c, 0xffffffff, 0x00010006,
 390         0x9190, 0xffffffff, 0x00090008,
 391         0x9194, 0xffffffff, 0x00070000,
 392         0x9198, 0xffffffff, 0x00030002,
 393         0x919c, 0xffffffff, 0x00050004,
 394         0x91a8, 0xffffffff, 0x00010006,
 395         0x91ac, 0xffffffff, 0x00090008,
 396         0x91b0, 0xffffffff, 0x00070000,
 397         0x91b4, 0xffffffff, 0x00030002,
 398         0x91b8, 0xffffffff, 0x00050004,
 399         0x91c4, 0xffffffff, 0x00010006,
 400         0x91c8, 0xffffffff, 0x00090008,
 401         0x91cc, 0xffffffff, 0x00070000,
 402         0x91d0, 0xffffffff, 0x00030002,
 403         0x91d4, 0xffffffff, 0x00050004,
 404         0x91e0, 0xffffffff, 0x00010006,
 405         0x91e4, 0xffffffff, 0x00090008,
 406         0x91e8, 0xffffffff, 0x00000000,
 407         0x91ec, 0xffffffff, 0x00070000,
 408         0x91f0, 0xffffffff, 0x00030002,
 409         0x91f4, 0xffffffff, 0x00050004,
 410         0x9200, 0xffffffff, 0x00010006,
 411         0x9204, 0xffffffff, 0x00090008,
 412         0x9208, 0xffffffff, 0x00070000,
 413         0x920c, 0xffffffff, 0x00030002,
 414         0x9210, 0xffffffff, 0x00050004,
 415         0x921c, 0xffffffff, 0x00010006,
 416         0x9220, 0xffffffff, 0x00090008,
 417         0x9224, 0xffffffff, 0x00070000,
 418         0x9228, 0xffffffff, 0x00030002,
 419         0x922c, 0xffffffff, 0x00050004,
 420         0x9238, 0xffffffff, 0x00010006,
 421         0x923c, 0xffffffff, 0x00090008,
 422         0x9240, 0xffffffff, 0x00070000,
 423         0x9244, 0xffffffff, 0x00030002,
 424         0x9248, 0xffffffff, 0x00050004,
 425         0x9254, 0xffffffff, 0x00010006,
 426         0x9258, 0xffffffff, 0x00090008,
 427         0x925c, 0xffffffff, 0x00070000,
 428         0x9260, 0xffffffff, 0x00030002,
 429         0x9264, 0xffffffff, 0x00050004,
 430         0x9270, 0xffffffff, 0x00010006,
 431         0x9274, 0xffffffff, 0x00090008,
 432         0x9278, 0xffffffff, 0x00070000,
 433         0x927c, 0xffffffff, 0x00030002,
 434         0x9280, 0xffffffff, 0x00050004,
 435         0x928c, 0xffffffff, 0x00010006,
 436         0x9290, 0xffffffff, 0x00090008,
 437         0x9294, 0xffffffff, 0x00000000,
 438         0x929c, 0xffffffff, 0x00000001,
 439         0x802c, 0xffffffff, 0xc0000000
 440 };
 441
 442 static const u32 redwood_mgcg_init[] =
 443 {
 444         0x802c, 0xffffffff, 0xc0000000,
 445         0x5448, 0xffffffff, 0x00000100,
 446         0x55e4, 0xffffffff, 0x00000100,
 447         0x160c, 0xffffffff, 0x00000100,
 448         0x5644, 0xffffffff, 0x00000100,
 449         0xc164, 0xffffffff, 0x00000100,
 450         0x8a18, 0xffffffff, 0x00000100,
 451         0x897c, 0xffffffff, 0x06000100,
 452         0x8b28, 0xffffffff, 0x00000100,
 453         0x9144, 0xffffffff, 0x00000100,
 454         0x9a60, 0xffffffff, 0x00000100,
 455         0x9868, 0xffffffff, 0x00000100,
 456         0x8d58, 0xffffffff, 0x00000100,
 457         0x9510, 0xffffffff, 0x00000100,
 458         0x949c, 0xffffffff, 0x00000100,
 459         0x9654, 0xffffffff, 0x00000100,
 460         0x9030, 0xffffffff, 0x00000100,
 461         0x9034, 0xffffffff, 0x00000100,
 462         0x9038, 0xffffffff, 0x00000100,
 463         0x903c, 0xffffffff, 0x00000100,
 464         0x9040, 0xffffffff, 0x00000100,
 465         0xa200, 0xffffffff, 0x00000100,
 466         0xa204, 0xffffffff, 0x00000100,
 467         0xa208, 0xffffffff, 0x00000100,
 468         0xa20c, 0xffffffff, 0x00000100,
 469         0x971c, 0xffffffff, 0x00000100,
 470         0x977c, 0xffffffff, 0x00000100,
 471         0x3f80, 0xffffffff, 0x00000100,
 472         0xa210, 0xffffffff, 0x00000100,
 473         0xa214, 0xffffffff, 0x00000100,
 474         0x4d8, 0xffffffff, 0x00000100,
 475         0x9784, 0xffffffff, 0x00000100,
 476         0x9698, 0xffffffff, 0x00000100,
 477         0x4d4, 0xffffffff, 0x00000200,
 478         0x30cc, 0xffffffff, 0x00000100,
 479         0xd0c0, 0xffffffff, 0xff000100,
 480         0x802c, 0xffffffff, 0x40000000,
 481         0x915c, 0xffffffff, 0x00010000,
 482         0x9160, 0xffffffff, 0x00030002,
 483         0x9178, 0xffffffff, 0x00070000,
 484         0x917c, 0xffffffff, 0x00030002,
 485         0x9180, 0xffffffff, 0x00050004,
 486         0x918c, 0xffffffff, 0x00010006,
 487         0x9190, 0xffffffff, 0x00090008,
 488         0x9194, 0xffffffff, 0x00070000,
 489         0x9198, 0xffffffff, 0x00030002,
 490         0x919c, 0xffffffff, 0x00050004,
 491         0x91a8, 0xffffffff, 0x00010006,
 492         0x91ac, 0xffffffff, 0x00090008,
 493         0x91b0, 0xffffffff, 0x00070000,
 494         0x91b4, 0xffffffff, 0x00030002,
 495         0x91b8, 0xffffffff, 0x00050004,
 496         0x91c4, 0xffffffff, 0x00010006,
 497         0x91c8, 0xffffffff, 0x00090008,
 498         0x91cc, 0xffffffff, 0x00070000,
 499         0x91d0, 0xffffffff, 0x00030002,
 500         0x91d4, 0xffffffff, 0x00050004,
 501         0x91e0, 0xffffffff, 0x00010006,
 502         0x91e4, 0xffffffff, 0x00090008,
 503         0x91e8, 0xffffffff, 0x00000000,
 504         0x91ec, 0xffffffff, 0x00070000,
 505         0x91f0, 0xffffffff, 0x00030002,
 506         0x91f4, 0xffffffff, 0x00050004,
 507         0x9200, 0xffffffff, 0x00010006,
 508         0x9204, 0xffffffff, 0x00090008,
 509         0x9294, 0xffffffff, 0x00000000,
 510         0x929c, 0xffffffff, 0x00000001,
 511         0x802c, 0xffffffff, 0xc0000000
 512 };
 513
 514 static const u32 cedar_golden_registers[] =
 515 {
 516         0x3f90, 0xffff0000, 0xff000000,
 517         0x9148, 0xffff0000, 0xff000000,
 518         0x3f94, 0xffff0000, 0xff000000,
 519         0x914c, 0xffff0000, 0xff000000,
 520         0x9b7c, 0xffffffff, 0x00000000,
 521         0x8a14, 0xffffffff, 0x00000007,
 522         0x8b10, 0xffffffff, 0x00000000,
 523         0x960c, 0xffffffff, 0x54763210,
 524         0x88c4, 0xffffffff, 0x000000c2,
 525         0x88d4, 0xffffffff, 0x00000000,
 526         0x8974, 0xffffffff, 0x00000000,
 527         0xc78, 0x00000080, 0x00000080,
 528         0x5eb4, 0xffffffff, 0x00000002,
 529         0x5e78, 0xffffffff, 0x001000f0,
 530         0x6104, 0x01000300, 0x00000000,
 531         0x5bc0, 0x00300000, 0x00000000,
 532         0x7030, 0xffffffff, 0x00000011,
 533         0x7c30, 0xffffffff, 0x00000011,
 534         0x10830, 0xffffffff, 0x00000011,
 535         0x11430, 0xffffffff, 0x00000011,
 536         0xd02c, 0xffffffff, 0x08421000,
 537         0x240c, 0xffffffff, 0x00000380,
 538         0x8b24, 0xffffffff, 0x00ff0fff,
 539         0x28a4c, 0x06000000, 0x06000000,
 540         0x10c, 0x00000001, 0x00000001,
 541         0x8d00, 0xffffffff, 0x100e4848,
 542         0x8d04, 0xffffffff, 0x00164745,
 543         0x8c00, 0xffffffff, 0xe4000003,
 544         0x8c04, 0xffffffff, 0x40600060,
 545         0x8c08, 0xffffffff, 0x001c001c,
 546         0x8cf0, 0xffffffff, 0x08e00410,
 547         0x8c20, 0xffffffff, 0x00800080,
 548         0x8c24, 0xffffffff, 0x00800080,
 549         0x8c18, 0xffffffff, 0x20202078,
 550         0x8c1c, 0xffffffff, 0x00001010,
 551         0x28350, 0xffffffff, 0x00000000,
 552         0xa008, 0xffffffff, 0x00010000,
 553         0x5c4, 0xffffffff, 0x00000001,
 554         0x9508, 0xffffffff, 0x00000002
 555 };
 556
 557 static const u32 cedar_mgcg_init[] =
 558 {
 559         0x802c, 0xffffffff, 0xc0000000,
 560         0x5448, 0xffffffff, 0x00000100,
 561         0x55e4, 0xffffffff, 0x00000100,
 562         0x160c, 0xffffffff, 0x00000100,
 563         0x5644, 0xffffffff, 0x00000100,
 564         0xc164, 0xffffffff, 0x00000100,
 565         0x8a18, 0xffffffff, 0x00000100,
 566         0x897c, 0xffffffff, 0x06000100,
 567         0x8b28, 0xffffffff, 0x00000100,
 568         0x9144, 0xffffffff, 0x00000100,
 569         0x9a60, 0xffffffff, 0x00000100,
 570         0x9868, 0xffffffff, 0x00000100,
 571         0x8d58, 0xffffffff, 0x00000100,
 572         0x9510, 0xffffffff, 0x00000100,
 573         0x949c, 0xffffffff, 0x00000100,
 574         0x9654, 0xffffffff, 0x00000100,
 575         0x9030, 0xffffffff, 0x00000100,
 576         0x9034, 0xffffffff, 0x00000100,
 577         0x9038, 0xffffffff, 0x00000100,
 578         0x903c, 0xffffffff, 0x00000100,
 579         0x9040, 0xffffffff, 0x00000100,
 580         0xa200, 0xffffffff, 0x00000100,
 581         0xa204, 0xffffffff, 0x00000100,
 582         0xa208, 0xffffffff, 0x00000100,
 583         0xa20c, 0xffffffff, 0x00000100,
 584         0x971c, 0xffffffff, 0x00000100,
 585         0x977c, 0xffffffff, 0x00000100,
 586         0x3f80, 0xffffffff, 0x00000100,
 587         0xa210, 0xffffffff, 0x00000100,
 588         0xa214, 0xffffffff, 0x00000100,
 589         0x4d8, 0xffffffff, 0x00000100,
 590         0x9784, 0xffffffff, 0x00000100,
 591         0x9698, 0xffffffff, 0x00000100,
 592         0x4d4, 0xffffffff, 0x00000200,
 593         0x30cc, 0xffffffff, 0x00000100,
 594         0xd0c0, 0xffffffff, 0xff000100,
 595         0x802c, 0xffffffff, 0x40000000,
 596         0x915c, 0xffffffff, 0x00010000,
 597         0x9178, 0xffffffff, 0x00050000,
 598         0x917c, 0xffffffff, 0x00030002,
 599         0x918c, 0xffffffff, 0x00010004,
 600         0x9190, 0xffffffff, 0x00070006,
 601         0x9194, 0xffffffff, 0x00050000,
 602         0x9198, 0xffffffff, 0x00030002,
 603         0x91a8, 0xffffffff, 0x00010004,
 604         0x91ac, 0xffffffff, 0x00070006,
 605         0x91e8, 0xffffffff, 0x00000000,
 606         0x9294, 0xffffffff, 0x00000000,
 607         0x929c, 0xffffffff, 0x00000001,
 608         0x802c, 0xffffffff, 0xc0000000
 609 };
 610
 611 static const u32 juniper_mgcg_init[] =
 612 {
 613         0x802c, 0xffffffff, 0xc0000000,
 614         0x5448, 0xffffffff, 0x00000100,
 615         0x55e4, 0xffffffff, 0x00000100,
 616         0x160c, 0xffffffff, 0x00000100,
 617         0x5644, 0xffffffff, 0x00000100,
 618         0xc164, 0xffffffff, 0x00000100,
 619         0x8a18, 0xffffffff, 0x00000100,
 620         0x897c, 0xffffffff, 0x06000100,
 621         0x8b28, 0xffffffff, 0x00000100,
 622         0x9144, 0xffffffff, 0x00000100,
 623         0x9a60, 0xffffffff, 0x00000100,
 624         0x9868, 0xffffffff, 0x00000100,
 625         0x8d58, 0xffffffff, 0x00000100,
 626         0x9510, 0xffffffff, 0x00000100,
 627         0x949c, 0xffffffff, 0x00000100,
 628         0x9654, 0xffffffff, 0x00000100,
 629         0x9030, 0xffffffff, 0x00000100,
 630         0x9034, 0xffffffff, 0x00000100,
 631         0x9038, 0xffffffff, 0x00000100,
 632         0x903c, 0xffffffff, 0x00000100,
 633         0x9040, 0xffffffff, 0x00000100,
 634         0xa200, 0xffffffff, 0x00000100,
 635         0xa204, 0xffffffff, 0x00000100,
 636         0xa208, 0xffffffff, 0x00000100,
 637         0xa20c, 0xffffffff, 0x00000100,
 638         0x971c, 0xffffffff, 0x00000100,
 639         0xd0c0, 0xffffffff, 0xff000100,
 640         0x802c, 0xffffffff, 0x40000000,
 641         0x915c, 0xffffffff, 0x00010000,
 642         0x9160, 0xffffffff, 0x00030002,
 643         0x9178, 0xffffffff, 0x00070000,
 644         0x917c, 0xffffffff, 0x00030002,
 645         0x9180, 0xffffffff, 0x00050004,
 646         0x918c, 0xffffffff, 0x00010006,
 647         0x9190, 0xffffffff, 0x00090008,
 648         0x9194, 0xffffffff, 0x00070000,
 649         0x9198, 0xffffffff, 0x00030002,
 650         0x919c, 0xffffffff, 0x00050004,
 651         0x91a8, 0xffffffff, 0x00010006,
 652         0x91ac, 0xffffffff, 0x00090008,
 653         0x91b0, 0xffffffff, 0x00070000,
 654         0x91b4, 0xffffffff, 0x00030002,
 655         0x91b8, 0xffffffff, 0x00050004,
 656         0x91c4, 0xffffffff, 0x00010006,
 657         0x91c8, 0xffffffff, 0x00090008,
 658         0x91cc, 0xffffffff, 0x00070000,
 659         0x91d0, 0xffffffff, 0x00030002,
 660         0x91d4, 0xffffffff, 0x00050004,
 661         0x91e0, 0xffffffff, 0x00010006,
 662         0x91e4, 0xffffffff, 0x00090008,
 663         0x91e8, 0xffffffff, 0x00000000,
 664         0x91ec, 0xffffffff, 0x00070000,
 665         0x91f0, 0xffffffff, 0x00030002,
 666         0x91f4, 0xffffffff, 0x00050004,
 667         0x9200, 0xffffffff, 0x00010006,
 668         0x9204, 0xffffffff, 0x00090008,
 669         0x9208, 0xffffffff, 0x00070000,
 670         0x920c, 0xffffffff, 0x00030002,
 671         0x9210, 0xffffffff, 0x00050004,
 672         0x921c, 0xffffffff, 0x00010006,
 673         0x9220, 0xffffffff, 0x00090008,
 674         0x9224, 0xffffffff, 0x00070000,
 675         0x9228, 0xffffffff, 0x00030002,
 676         0x922c, 0xffffffff, 0x00050004,
 677         0x9238, 0xffffffff, 0x00010006,
 678         0x923c, 0xffffffff, 0x00090008,
 679         0x9240, 0xffffffff, 0x00070000,
 680         0x9244, 0xffffffff, 0x00030002,
 681         0x9248, 0xffffffff, 0x00050004,
 682         0x9254, 0xffffffff, 0x00010006,
 683         0x9258, 0xffffffff, 0x00090008,
 684         0x925c, 0xffffffff, 0x00070000,
 685         0x9260, 0xffffffff, 0x00030002,
 686         0x9264, 0xffffffff, 0x00050004,
 687         0x9270, 0xffffffff, 0x00010006,
 688         0x9274, 0xffffffff, 0x00090008,
 689         0x9278, 0xffffffff, 0x00070000,
 690         0x927c, 0xffffffff, 0x00030002,
 691         0x9280, 0xffffffff, 0x00050004,
 692         0x928c, 0xffffffff, 0x00010006,
 693         0x9290, 0xffffffff, 0x00090008,
 694         0x9294, 0xffffffff, 0x00000000,
 695         0x929c, 0xffffffff, 0x00000001,
 696         0x802c, 0xffffffff, 0xc0000000,
 697         0x977c, 0xffffffff, 0x00000100,
 698         0x3f80, 0xffffffff, 0x00000100,
 699         0xa210, 0xffffffff, 0x00000100,
 700         0xa214, 0xffffffff, 0x00000100,
 701         0x4d8, 0xffffffff, 0x00000100,
 702         0x9784, 0xffffffff, 0x00000100,
 703         0x9698, 0xffffffff, 0x00000100,
 704         0x4d4, 0xffffffff, 0x00000200,
 705         0x30cc, 0xffffffff, 0x00000100,
 706         0x802c, 0xffffffff, 0xc0000000
 707 };
 708
 709 static const u32 supersumo_golden_registers[] =
 710 {
 711         0x5eb4, 0xffffffff, 0x00000002,
 712         0x5c4, 0xffffffff, 0x00000001,
 713         0x7030, 0xffffffff, 0x00000011,
 714         0x7c30, 0xffffffff, 0x00000011,
 715         0x6104, 0x01000300, 0x00000000,
 716         0x5bc0, 0x00300000, 0x00000000,
 717         0x8c04, 0xffffffff, 0x40600060,
 718         0x8c08, 0xffffffff, 0x001c001c,
 719         0x8c20, 0xffffffff, 0x00800080,
 720         0x8c24, 0xffffffff, 0x00800080,
 721         0x8c18, 0xffffffff, 0x20202078,
 722         0x8c1c, 0xffffffff, 0x00001010,
 723         0x918c, 0xffffffff, 0x00010006,
 724         0x91a8, 0xffffffff, 0x00010006,
 725         0x91c4, 0xffffffff, 0x00010006,
 726         0x91e0, 0xffffffff, 0x00010006,
 727         0x9200, 0xffffffff, 0x00010006,
 728         0x9150, 0xffffffff, 0x6e944040,
 729         0x917c, 0xffffffff, 0x00030002,
 730         0x9180, 0xffffffff, 0x00050004,
 731         0x9198, 0xffffffff, 0x00030002,
 732         0x919c, 0xffffffff, 0x00050004,
 733         0x91b4, 0xffffffff, 0x00030002,
 734         0x91b8, 0xffffffff, 0x00050004,
 735         0x91d0, 0xffffffff, 0x00030002,
 736         0x91d4, 0xffffffff, 0x00050004,
 737         0x91f0, 0xffffffff, 0x00030002,
 738         0x91f4, 0xffffffff, 0x00050004,
 739         0x915c, 0xffffffff, 0x00010000,
 740         0x9160, 0xffffffff, 0x00030002,
 741         0x3f90, 0xffff0000, 0xff000000,
 742         0x9178, 0xffffffff, 0x00070000,
 743         0x9194, 0xffffffff, 0x00070000,
 744         0x91b0, 0xffffffff, 0x00070000,
 745         0x91cc, 0xffffffff, 0x00070000,
 746         0x91ec, 0xffffffff, 0x00070000,
 747         0x9148, 0xffff0000, 0xff000000,
 748         0x9190, 0xffffffff, 0x00090008,
 749         0x91ac, 0xffffffff, 0x00090008,
 750         0x91c8, 0xffffffff, 0x00090008,
 751         0x91e4, 0xffffffff, 0x00090008,
 752         0x9204, 0xffffffff, 0x00090008,
 753         0x3f94, 0xffff0000, 0xff000000,
 754         0x914c, 0xffff0000, 0xff000000,
 755         0x929c, 0xffffffff, 0x00000001,
 756         0x8a18, 0xffffffff, 0x00000100,
 757         0x8b28, 0xffffffff, 0x00000100,
 758         0x9144, 0xffffffff, 0x00000100,
 759         0x5644, 0xffffffff, 0x00000100,
 760         0x9b7c, 0xffffffff, 0x00000000,
 761         0x8030, 0xffffffff, 0x0000100a,
 762         0x8a14, 0xffffffff, 0x00000007,
 763         0x8b24, 0xffffffff, 0x00ff0fff,
 764         0x8b10, 0xffffffff, 0x00000000,
 765         0x28a4c, 0x06000000, 0x06000000,
 766         0x4d8, 0xffffffff, 0x00000100,
 767         0x913c, 0xffff000f, 0x0100000a,
 768         0x960c, 0xffffffff, 0x54763210,
 769         0x88c4, 0xffffffff, 0x000000c2,
 770         0x88d4, 0xffffffff, 0x00000010,
 771         0x8974, 0xffffffff, 0x00000000,
 772         0xc78, 0x00000080, 0x00000080,
 773         0x5e78, 0xffffffff, 0x001000f0,
 774         0xd02c, 0xffffffff, 0x08421000,
 775         0xa008, 0xffffffff, 0x00010000,
 776         0x8d00, 0xffffffff, 0x100e4848,
 777         0x8d04, 0xffffffff, 0x00164745,
 778         0x8c00, 0xffffffff, 0xe4000003,
 779         0x8cf0, 0x1fffffff, 0x08e00620,
 780         0x28350, 0xffffffff, 0x00000000,
 781         0x9508, 0xffffffff, 0x00000002
 782 };
 783
 784 static const u32 sumo_golden_registers[] =
 785 {
 786         0x900c, 0x00ffffff, 0x0017071f,
 787         0x8c18, 0xffffffff, 0x10101060,
 788         0x8c1c, 0xffffffff, 0x00001010,
 789         0x8c30, 0x0000000f, 0x00000005,
 790         0x9688, 0x0000000f, 0x00000007
 791 };
 792
 793 static const u32 wrestler_golden_registers[] =
 794 {
 795         0x5eb4, 0xffffffff, 0x00000002,
 796         0x5c4, 0xffffffff, 0x00000001,
 797         0x7030, 0xffffffff, 0x00000011,
 798         0x7c30, 0xffffffff, 0x00000011,
 799         0x6104, 0x01000300, 0x00000000,
 800         0x5bc0, 0x00300000, 0x00000000,
 801         0x918c, 0xffffffff, 0x00010006,
 802         0x91a8, 0xffffffff, 0x00010006,
 803         0x9150, 0xffffffff, 0x6e944040,
 804         0x917c, 0xffffffff, 0x00030002,
 805         0x9198, 0xffffffff, 0x00030002,
 806         0x915c, 0xffffffff, 0x00010000,
 807         0x3f90, 0xffff0000, 0xff000000,
 808         0x9178, 0xffffffff, 0x00070000,
 809         0x9194, 0xffffffff, 0x00070000,
 810         0x9148, 0xffff0000, 0xff000000,
 811         0x9190, 0xffffffff, 0x00090008,
 812         0x91ac, 0xffffffff, 0x00090008,
 813         0x3f94, 0xffff0000, 0xff000000,
 814         0x914c, 0xffff0000, 0xff000000,
 815         0x929c, 0xffffffff, 0x00000001,
 816         0x8a18, 0xffffffff, 0x00000100,
 817         0x8b28, 0xffffffff, 0x00000100,
 818         0x9144, 0xffffffff, 0x00000100,
 819         0x9b7c, 0xffffffff, 0x00000000,
 820         0x8030, 0xffffffff, 0x0000100a,
 821         0x8a14, 0xffffffff, 0x00000001,
 822         0x8b24, 0xffffffff, 0x00ff0fff,
 823         0x8b10, 0xffffffff, 0x00000000,
 824         0x28a4c, 0x06000000, 0x06000000,
 825         0x4d8, 0xffffffff, 0x00000100,
 826         0x913c, 0xffff000f, 0x0100000a,
 827         0x960c, 0xffffffff, 0x54763210,
 828         0x88c4, 0xffffffff, 0x000000c2,
 829         0x88d4, 0xffffffff, 0x00000010,
 830         0x8974, 0xffffffff, 0x00000000,
 831         0xc78, 0x00000080, 0x00000080,
 832         0x5e78, 0xffffffff, 0x001000f0,
 833         0xd02c, 0xffffffff, 0x08421000,
 834         0xa008, 0xffffffff, 0x00010000,
 835         0x8d00, 0xffffffff, 0x100e4848,
 836         0x8d04, 0xffffffff, 0x00164745,
 837         0x8c00, 0xffffffff, 0xe4000003,
 838         0x8cf0, 0x1fffffff, 0x08e00410,
 839         0x28350, 0xffffffff, 0x00000000,
 840         0x9508, 0xffffffff, 0x00000002,
 841         0x900c, 0xffffffff, 0x0017071f,
 842         0x8c18, 0xffffffff, 0x10101060,
 843         0x8c1c, 0xffffffff, 0x00001010
 844 };
 845
 846 static const u32 barts_golden_registers[] =
 847 {
 848         0x5eb4, 0xffffffff, 0x00000002,
 849         0x5e78, 0x8f311ff1, 0x001000f0,
 850         0x3f90, 0xffff0000, 0xff000000,
 851         0x9148, 0xffff0000, 0xff000000,
 852         0x3f94, 0xffff0000, 0xff000000,
 853         0x914c, 0xffff0000, 0xff000000,
 854         0xc78, 0x00000080, 0x00000080,
 855         0xbd4, 0x70073777, 0x00010001,
 856         0xd02c, 0xbfffff1f, 0x08421000,
 857         0xd0b8, 0x03773777, 0x02011003,
 858         0x5bc0, 0x00200000, 0x50100000,
 859         0x98f8, 0x33773777, 0x02011003,
 860         0x98fc, 0xffffffff, 0x76543210,
 861         0x7030, 0x31000311, 0x00000011,
 862         0x2f48, 0x00000007, 0x02011003,
 863         0x6b28, 0x00000010, 0x00000012,
 864         0x7728, 0x00000010, 0x00000012,
 865         0x10328, 0x00000010, 0x00000012,
 866         0x10f28, 0x00000010, 0x00000012,
 867         0x11b28, 0x00000010, 0x00000012,
 868         0x12728, 0x00000010, 0x00000012,
 869         0x240c, 0x000007ff, 0x00000380,
 870         0x8a14, 0xf000001f, 0x00000007,
 871         0x8b24, 0x3fff3fff, 0x00ff0fff,
 872         0x8b10, 0x0000ff0f, 0x00000000,
 873         0x28a4c, 0x07ffffff, 0x06000000,
 874         0x10c, 0x00000001, 0x00010003,
 875         0xa02c, 0xffffffff, 0x0000009b,
 876         0x913c, 0x0000000f, 0x0100000a,
 877         0x8d00, 0xffff7f7f, 0x100e4848,
 878         0x8d04, 0x00ffffff, 0x00164745,
 879         0x8c00, 0xfffc0003, 0xe4000003,
 880         0x8c04, 0xf8ff00ff, 0x40600060,
 881         0x8c08, 0x00ff00ff, 0x001c001c,
 882         0x8cf0, 0x1fff1fff, 0x08e00620,
 883         0x8c20, 0x0fff0fff, 0x00800080,
 884         0x8c24, 0x0fff0fff, 0x00800080,
 885         0x8c18, 0xffffffff, 0x20202078,
 886         0x8c1c, 0x0000ffff, 0x00001010,
 887         0x28350, 0x00000f01, 0x00000000,
 888         0x9508, 0x3700001f, 0x00000002,
 889         0x960c, 0xffffffff, 0x54763210,
 890         0x88c4, 0x001f3ae3, 0x000000c2,
 891         0x88d4, 0x0000001f, 0x00000010,
 892         0x8974, 0xffffffff, 0x00000000
 893 };
 894
 895 static const u32 turks_golden_registers[] =
 896 {
 897         0x5eb4, 0xffffffff, 0x00000002,
 898         0x5e78, 0x8f311ff1, 0x001000f0,
 899         0x8c8, 0x00003000, 0x00001070,
 900         0x8cc, 0x000fffff, 0x00040035,
 901         0x3f90, 0xffff0000, 0xfff00000,
 902         0x9148, 0xffff0000, 0xfff00000,
 903         0x3f94, 0xffff0000, 0xfff00000,
 904         0x914c, 0xffff0000, 0xfff00000,
 905         0xc78, 0x00000080, 0x00000080,
 906         0xbd4, 0x00073007, 0x00010002,
 907         0xd02c, 0xbfffff1f, 0x08421000,
 908         0xd0b8, 0x03773777, 0x02010002,
 909         0x5bc0, 0x00200000, 0x50100000,
 910         0x98f8, 0x33773777, 0x00010002,
 911         0x98fc, 0xffffffff, 0x33221100,
 912         0x7030, 0x31000311, 0x00000011,
 913         0x2f48, 0x33773777, 0x00010002,
 914         0x6b28, 0x00000010, 0x00000012,
 915         0x7728, 0x00000010, 0x00000012,
 916         0x10328, 0x00000010, 0x00000012,
 917         0x10f28, 0x00000010, 0x00000012,
 918         0x11b28, 0x00000010, 0x00000012,
 919         0x12728, 0x00000010, 0x00000012,
 920         0x240c, 0x000007ff, 0x00000380,
 921         0x8a14, 0xf000001f, 0x00000007,
 922         0x8b24, 0x3fff3fff, 0x00ff0fff,
 923         0x8b10, 0x0000ff0f, 0x00000000,
 924         0x28a4c, 0x07ffffff, 0x06000000,
 925         0x10c, 0x00000001, 0x00010003,
 926         0xa02c, 0xffffffff, 0x0000009b,
 927         0x913c, 0x0000000f, 0x0100000a,
 928         0x8d00, 0xffff7f7f, 0x100e4848,
 929         0x8d04, 0x00ffffff, 0x00164745,
 930         0x8c00, 0xfffc0003, 0xe4000003,
 931         0x8c04, 0xf8ff00ff, 0x40600060,
 932         0x8c08, 0x00ff00ff, 0x001c001c,
 933         0x8cf0, 0x1fff1fff, 0x08e00410,
 934         0x8c20, 0x0fff0fff, 0x00800080,
 935         0x8c24, 0x0fff0fff, 0x00800080,
 936         0x8c18, 0xffffffff, 0x20202078,
 937         0x8c1c, 0x0000ffff, 0x00001010,
 938         0x28350, 0x00000f01, 0x00000000,
 939         0x9508, 0x3700001f, 0x00000002,
 940         0x960c, 0xffffffff, 0x54763210,
 941         0x88c4, 0x001f3ae3, 0x000000c2,
 942         0x88d4, 0x0000001f, 0x00000010,
 943         0x8974, 0xffffffff, 0x00000000
 944 };
 945
 946 static const u32 caicos_golden_registers[] =
 947 {
 948         0x5eb4, 0xffffffff, 0x00000002,
 949         0x5e78, 0x8f311ff1, 0x001000f0,
 950         0x8c8, 0x00003420, 0x00001450,
 951         0x8cc, 0x000fffff, 0x00040035,
 952         0x3f90, 0xffff0000, 0xfffc0000,
 953         0x9148, 0xffff0000, 0xfffc0000,
 954         0x3f94, 0xffff0000, 0xfffc0000,
 955         0x914c, 0xffff0000, 0xfffc0000,
 956         0xc78, 0x00000080, 0x00000080,
 957         0xbd4, 0x00073007, 0x00010001,
 958         0xd02c, 0xbfffff1f, 0x08421000,
 959         0xd0b8, 0x03773777, 0x02010001,
 960         0x5bc0, 0x00200000, 0x50100000,
 961         0x98f8, 0x33773777, 0x02010001,
 962         0x98fc, 0xffffffff, 0x33221100,
 963         0x7030, 0x31000311, 0x00000011,
 964         0x2f48, 0x33773777, 0x02010001,
 965         0x6b28, 0x00000010, 0x00000012,
 966         0x7728, 0x00000010, 0x00000012,
 967         0x10328, 0x00000010, 0x00000012,
 968         0x10f28, 0x00000010, 0x00000012,
 969         0x11b28, 0x00000010, 0x00000012,
 970         0x12728, 0x00000010, 0x00000012,
 971         0x240c, 0x000007ff, 0x00000380,
 972         0x8a14, 0xf000001f, 0x00000001,
 973         0x8b24, 0x3fff3fff, 0x00ff0fff,
 974         0x8b10, 0x0000ff0f, 0x00000000,
 975         0x28a4c, 0x07ffffff, 0x06000000,
 976         0x10c, 0x00000001, 0x00010003,
 977         0xa02c, 0xffffffff, 0x0000009b,
 978         0x913c, 0x0000000f, 0x0100000a,
 979         0x8d00, 0xffff7f7f, 0x100e4848,
 980         0x8d04, 0x00ffffff, 0x00164745,
 981         0x8c00, 0xfffc0003, 0xe4000003,
 982         0x8c04, 0xf8ff00ff, 0x40600060,
 983         0x8c08, 0x00ff00ff, 0x001c001c,
 984         0x8cf0, 0x1fff1fff, 0x08e00410,
 985         0x8c20, 0x0fff0fff, 0x00800080,
 986         0x8c24, 0x0fff0fff, 0x00800080,
 987         0x8c18, 0xffffffff, 0x20202078,
 988         0x8c1c, 0x0000ffff, 0x00001010,
 989         0x28350, 0x00000f01, 0x00000000,
 990         0x9508, 0x3700001f, 0x00000002,
 991         0x960c, 0xffffffff, 0x54763210,
 992         0x88c4, 0x001f3ae3, 0x000000c2,
 993         0x88d4, 0x0000001f, 0x00000010,
 994         0x8974, 0xffffffff, 0x00000000
 995 };
 996
 997 static void evergreen_init_golden_registers(struct radeon_device *rdev)
 998 {
 999         switch (rdev->family) {
1000         case CHIP_CYPRESS:
1001         case CHIP_HEMLOCK:
1002                 radeon_program_register_sequence(rdev,
1003                                                  evergreen_golden_registers,
1004                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers));
1005                 radeon_program_register_sequence(rdev,
1006                                                  evergreen_golden_registers2,
1007                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1008                 radeon_program_register_sequence(rdev,
1009                                                  cypress_mgcg_init,
1010                                                  (const u32)ARRAY_SIZE(cypress_mgcg_init));
1011                 break;
1012         case CHIP_JUNIPER:
1013                 radeon_program_register_sequence(rdev,
1014                                                  evergreen_golden_registers,
1015                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers));
1016                 radeon_program_register_sequence(rdev,
1017                                                  evergreen_golden_registers2,
1018                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1019                 radeon_program_register_sequence(rdev,
1020                                                  juniper_mgcg_init,
1021                                                  (const u32)ARRAY_SIZE(juniper_mgcg_init));
1022                 break;
1023         case CHIP_REDWOOD:
1024                 radeon_program_register_sequence(rdev,
1025                                                  evergreen_golden_registers,
1026                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers));
1027                 radeon_program_register_sequence(rdev,
1028                                                  evergreen_golden_registers2,
1029                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1030                 radeon_program_register_sequence(rdev,
1031                                                  redwood_mgcg_init,
1032                                                  (const u32)ARRAY_SIZE(redwood_mgcg_init));
1033                 break;
1034         case CHIP_CEDAR:
1035                 radeon_program_register_sequence(rdev,
1036                                                  cedar_golden_registers,
1037                                                  (const u32)ARRAY_SIZE(cedar_golden_registers));
1038                 radeon_program_register_sequence(rdev,
1039                                                  evergreen_golden_registers2,
1040                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1041                 radeon_program_register_sequence(rdev,
1042                                                  cedar_mgcg_init,
1043                                                  (const u32)ARRAY_SIZE(cedar_mgcg_init));
1044                 break;
1045         case CHIP_PALM:
1046                 radeon_program_register_sequence(rdev,
1047                                                  wrestler_golden_registers,
1048                                                  (const u32)ARRAY_SIZE(wrestler_golden_registers));
1049                 break;
1050         case CHIP_SUMO:
1051                 radeon_program_register_sequence(rdev,
1052                                                  supersumo_golden_registers,
1053                                                  (const u32)ARRAY_SIZE(supersumo_golden_registers));
1054                 break;
1055         case CHIP_SUMO2:
1056                 radeon_program_register_sequence(rdev,
1057                                                  supersumo_golden_registers,
1058                                                  (const u32)ARRAY_SIZE(supersumo_golden_registers));
1059                 radeon_program_register_sequence(rdev,
1060                                                  sumo_golden_registers,
1061                                                  (const u32)ARRAY_SIZE(sumo_golden_registers));
1062                 break;
1063         case CHIP_BARTS:
1064                 radeon_program_register_sequence(rdev,
1065                                                  barts_golden_registers,
1066                                                  (const u32)ARRAY_SIZE(barts_golden_registers));
1067                 break;
1068         case CHIP_TURKS:
1069                 radeon_program_register_sequence(rdev,
1070                                                  turks_golden_registers,
1071                                                  (const u32)ARRAY_SIZE(turks_golden_registers));
1072                 break;
1073         case CHIP_CAICOS:
1074                 radeon_program_register_sequence(rdev,
1075                                                  caicos_golden_registers,
1076                                                  (const u32)ARRAY_SIZE(caicos_golden_registers));
1077                 break;
1078         default:
1079                 break;
1080         }
1081 }
1082
1083 /**
1084  * evergreen_get_allowed_info_register - fetch the register for the info ioctl
1085  *
1086  * @rdev: radeon_device pointer
1087  * @reg: register offset in bytes
1088  * @val: register value
1089  *
1090  * Returns 0 for success or -EINVAL for an invalid register
1091  *
1092  */
1093 int evergreen_get_allowed_info_register(struct radeon_device *rdev,
1094                                         u32 reg, u32 *val)
1095 {
1096         switch (reg) {
1097         case GRBM_STATUS:
1098         case GRBM_STATUS_SE0:
1099         case GRBM_STATUS_SE1:
1100         case SRBM_STATUS:
1101         case SRBM_STATUS2:
1102         case DMA_STATUS_REG:
1103         case UVD_STATUS:
1104                 *val = RREG32(reg);
1105                 return 0;
1106         default:
1107                 return -EINVAL;
1108         }
1109 }
1110
1111 void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw,
1112                              unsigned *bankh, unsigned *mtaspect,
1113                              unsigned *tile_split)
1114 {
1115         *bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK;
1116         *bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK;
1117         *mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK;
1118         *tile_split = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK;
1119         switch (*bankw) {
1120         default:
1121         case 1: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_1; break;
1122         case 2: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_2; break;
1123         case 4: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_4; break;
1124         case 8: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_8; break;
1125         }
1126         switch (*bankh) {
1127         default:
1128         case 1: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_1; break;
1129         case 2: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_2; break;
1130         case 4: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_4; break;
1131         case 8: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_8; break;
1132         }
1133         switch (*mtaspect) {
1134         default:
1135         case 1: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1; break;
1136         case 2: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2; break;
1137         case 4: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4; break;
1138         case 8: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8; break;
1139         }
1140 }
1141
1142 static int sumo_set_uvd_clock(struct radeon_device *rdev, u32 clock,
1143                               u32 cntl_reg, u32 status_reg)
1144 {
1145         int r, i;
1146         struct atom_clock_dividers dividers;
1147
1148         r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
1149                                            clock, false, &dividers);
1150         if (r)
1151                 return r;
1152
1153         WREG32_P(cntl_reg, dividers.post_div, ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK));
1154
1155         for (i = 0; i < 100; i++) {
1156                 if (RREG32(status_reg) & DCLK_STATUS)
1157                         break;
1158                 mdelay(10);
1159         }
1160         if (i == 100)
1161                 return -ETIMEDOUT;
1162
1163         return 0;
1164 }
1165
1166 int sumo_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1167 {
1168         int r = 0;
1169         u32 cg_scratch = RREG32(CG_SCRATCH1);
1170
1171         r = sumo_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
1172         if (r)
1173                 goto done;
1174         cg_scratch &= 0xffff0000;
1175         cg_scratch |= vclk / 100; /* Mhz */
1176
1177         r = sumo_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
1178         if (r)
1179                 goto done;
1180         cg_scratch &= 0x0000ffff;
1181         cg_scratch |= (dclk / 100) << 16; /* Mhz */
1182
1183 done:
1184         WREG32(CG_SCRATCH1, cg_scratch);
1185
1186         return r;
1187 }
1188
1189 int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1190 {
1191         /* start off with something large */
1192         unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
1193         int r;
1194
1195         /* bypass vclk and dclk with bclk */
1196         WREG32_P(CG_UPLL_FUNC_CNTL_2,
1197                 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
1198                 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1199
1200         /* put PLL in bypass mode */
1201         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
1202
1203         if (!vclk || !dclk) {
1204                 /* keep the Bypass mode, put PLL to sleep */
1205                 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1206                 return 0;
1207         }
1208
1209         r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
1210                                           16384, 0x03FFFFFF, 0, 128, 5,
1211                                           &fb_div, &vclk_div, &dclk_div);
1212         if (r)
1213                 return r;
1214
1215         /* set VCO_MODE to 1 */
1216         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
1217
1218         /* toggle UPLL_SLEEP to 1 then back to 0 */
1219         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1220         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
1221
1222         /* deassert UPLL_RESET */
1223         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1224
1225         mdelay(1);
1226
1227         r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1228         if (r)
1229                 return r;
1230
1231         /* assert UPLL_RESET again */
1232         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
1233
1234         /* disable spread spectrum. */
1235         WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
1236
1237         /* set feedback divider */
1238         WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
1239
1240         /* set ref divider to 0 */
1241         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
1242
1243         if (fb_div < 307200)
1244                 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
1245         else
1246                 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
1247
1248         /* set PDIV_A and PDIV_B */
1249         WREG32_P(CG_UPLL_FUNC_CNTL_2,
1250                 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
1251                 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
1252
1253         /* give the PLL some time to settle */
1254         mdelay(15);
1255
1256         /* deassert PLL_RESET */
1257         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1258
1259         mdelay(15);
1260
1261         /* switch from bypass mode to normal mode */
1262         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
1263
1264         r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1265         if (r)
1266                 return r;
1267
1268         /* switch VCLK and DCLK selection */
1269         WREG32_P(CG_UPLL_FUNC_CNTL_2,
1270                 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
1271                 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1272
1273         mdelay(100);
1274
1275         return 0;
1276 }
1277
1278 void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
1279 {
1280         int readrq;
1281         u16 v;
1282
1283         readrq = pcie_get_readrq(rdev->pdev);
1284         v = ffs(readrq) - 8;
1285         /* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it
1286          * to avoid hangs or perfomance issues
1287          */
1288         if ((v == 0) || (v == 6) || (v == 7))
1289                 pcie_set_readrq(rdev->pdev, 512);
1290 }
1291
1292 void dce4_program_fmt(struct drm_encoder *encoder)
1293 {
1294         struct drm_device *dev = encoder->dev;
1295         struct radeon_device *rdev = dev->dev_private;
1296         struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
1297         struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
1298         struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
1299         int bpc = 0;
1300         u32 tmp = 0;
1301         enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
1302
1303         if (connector) {
1304                 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1305                 bpc = radeon_get_monitor_bpc(connector);
1306                 dither = radeon_connector->dither;
1307         }
1308
1309         /* LVDS/eDP FMT is set up by atom */
1310         if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
1311                 return;
1312
1313         /* not needed for analog */
1314         if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
1315             (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
1316                 return;
1317
1318         if (bpc == 0)
1319                 return;
1320
1321         switch (bpc) {
1322         case 6:
1323                 if (dither == RADEON_FMT_DITHER_ENABLE)
1324                         /* XXX sort out optimal dither settings */
1325                         tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1326                                 FMT_SPATIAL_DITHER_EN);
1327                 else
1328                         tmp |= FMT_TRUNCATE_EN;
1329                 break;
1330         case 8:
1331                 if (dither == RADEON_FMT_DITHER_ENABLE)
1332                         /* XXX sort out optimal dither settings */
1333                         tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1334                                 FMT_RGB_RANDOM_ENABLE |
1335                                 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH);
1336                 else
1337                         tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH);
1338                 break;
1339         case 10:
1340         default:
1341                 /* not needed */
1342                 break;
1343         }
1344
1345         WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
1346 }
1347
1348 static bool dce4_is_in_vblank(struct radeon_device *rdev, int crtc)
1349 {
1350         if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK)
1351                 return true;
1352         else
1353                 return false;
1354 }
1355
1356 static bool dce4_is_counter_moving(struct radeon_device *rdev, int crtc)
1357 {
1358         u32 pos1, pos2;
1359
1360         pos1 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1361         pos2 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1362
1363         if (pos1 != pos2)
1364                 return true;
1365         else
1366                 return false;
1367 }
1368
1369 /**
1370  * dce4_wait_for_vblank - vblank wait asic callback.
1371  *
1372  * @rdev: radeon_device pointer
1373  * @crtc: crtc to wait for vblank on
1374  *
1375  * Wait for vblank on the requested crtc (evergreen+).
1376  */
1377 void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc)
1378 {
1379         unsigned i = 0;
1380
1381         if (crtc >= rdev->num_crtc)
1382                 return;
1383
1384         if (!(RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN))
1385                 return;
1386
1387         /* depending on when we hit vblank, we may be close to active; if so,
1388          * wait for another frame.
1389          */
1390         while (dce4_is_in_vblank(rdev, crtc)) {
1391                 if (i++ % 100 == 0) {
1392                         if (!dce4_is_counter_moving(rdev, crtc))
1393                                 break;
1394                 }
1395         }
1396
1397         while (!dce4_is_in_vblank(rdev, crtc)) {
1398                 if (i++ % 100 == 0) {
1399                         if (!dce4_is_counter_moving(rdev, crtc))
1400                                 break;
1401                 }
1402         }
1403 }
1404
1405 /**
1406  * evergreen_page_flip - pageflip callback.
1407  *
1408  * @rdev: radeon_device pointer
1409  * @crtc_id: crtc to cleanup pageflip on
1410  * @crtc_base: new address of the crtc (GPU MC address)
1411  * @async: asynchronous flip
1412  *
1413  * Triggers the actual pageflip by updating the primary
1414  * surface base address (evergreen+).
1415  */
1416 void evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base,
1417                          bool async)
1418 {
1419         struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1420         struct drm_framebuffer *fb = radeon_crtc->base.primary->fb;
1421
1422         /* flip at hsync for async, default is vsync */
1423         WREG32(EVERGREEN_GRPH_FLIP_CONTROL + radeon_crtc->crtc_offset,
1424                async ? EVERGREEN_GRPH_SURFACE_UPDATE_H_RETRACE_EN : 0);
1425         /* update pitch */
1426         WREG32(EVERGREEN_GRPH_PITCH + radeon_crtc->crtc_offset,
1427                fb->pitches[0] / fb->format->cpp[0]);
1428         /* update the scanout addresses */
1429         WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1430                upper_32_bits(crtc_base));
1431         WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1432                (u32)crtc_base);
1433         /* post the write */
1434         RREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset);
1435 }
1436
1437 /**
1438  * evergreen_page_flip_pending - check if page flip is still pending
1439  *
1440  * @rdev: radeon_device pointer
1441  * @crtc_id: crtc to check
1442  *
1443  * Returns the current update pending status.
1444  */
1445 bool evergreen_page_flip_pending(struct radeon_device *rdev, int crtc_id)
1446 {
1447         struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1448
1449         /* Return current update_pending status: */
1450         return !!(RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) &
1451                 EVERGREEN_GRPH_SURFACE_UPDATE_PENDING);
1452 }
1453
1454 /* get temperature in millidegrees */
1455 int evergreen_get_temp(struct radeon_device *rdev)
1456 {
1457         u32 temp, toffset;
1458         int actual_temp = 0;
1459
1460         if (rdev->family == CHIP_JUNIPER) {
1461                 toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
1462                         TOFFSET_SHIFT;
1463                 temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
1464                         TS0_ADC_DOUT_SHIFT;
1465
1466                 if (toffset & 0x100)
1467                         actual_temp = temp / 2 - (0x200 - toffset);
1468                 else
1469                         actual_temp = temp / 2 + toffset;
1470
1471                 actual_temp = actual_temp * 1000;
1472
1473         } else {
1474                 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
1475                         ASIC_T_SHIFT;
1476
1477                 if (temp & 0x400)
1478                         actual_temp = -256;
1479                 else if (temp & 0x200)
1480                         actual_temp = 255;
1481                 else if (temp & 0x100) {
1482                         actual_temp = temp & 0x1ff;
1483                         actual_temp |= ~0x1ff;
1484                 } else
1485                         actual_temp = temp & 0xff;
1486
1487                 actual_temp = (actual_temp * 1000) / 2;
1488         }
1489
1490         return actual_temp;
1491 }
1492
1493 int sumo_get_temp(struct radeon_device *rdev)
1494 {
1495         u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
1496         int actual_temp = temp - 49;
1497
1498         return actual_temp * 1000;
1499 }
1500
1501 /**
1502  * sumo_pm_init_profile - Initialize power profiles callback.
1503  *
1504  * @rdev: radeon_device pointer
1505  *
1506  * Initialize the power states used in profile mode
1507  * (sumo, trinity, SI).
1508  * Used for profile mode only.
1509  */
1510 void sumo_pm_init_profile(struct radeon_device *rdev)
1511 {
1512         int idx;
1513
1514         /* default */
1515         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1516         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1517         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1518         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
1519
1520         /* low,mid sh/mh */
1521         if (rdev->flags & RADEON_IS_MOBILITY)
1522                 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1523         else
1524                 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1525
1526         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1527         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1528         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1529         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1530
1531         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1532         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1533         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1534         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1535
1536         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1537         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1538         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1539         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
1540
1541         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1542         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1543         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1544         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
1545
1546         /* high sh/mh */
1547         idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1548         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1549         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1550         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1551         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx =
1552                 rdev->pm.power_state[idx].num_clock_modes - 1;
1553
1554         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1555         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1556         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1557         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx =
1558                 rdev->pm.power_state[idx].num_clock_modes - 1;
1559 }
1560
1561 /**
1562  * btc_pm_init_profile - Initialize power profiles callback.
1563  *
1564  * @rdev: radeon_device pointer
1565  *
1566  * Initialize the power states used in profile mode
1567  * (BTC, cayman).
1568  * Used for profile mode only.
1569  */
1570 void btc_pm_init_profile(struct radeon_device *rdev)
1571 {
1572         int idx;
1573
1574         /* default */
1575         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1576         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1577         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1578         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
1579         /* starting with BTC, there is one state that is used for both
1580          * MH and SH.  Difference is that we always use the high clock index for
1581          * mclk.
1582          */
1583         if (rdev->flags & RADEON_IS_MOBILITY)
1584                 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1585         else
1586                 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1587         /* low sh */
1588         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1589         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1590         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1591         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1592         /* mid sh */
1593         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1594         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1595         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1596         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
1597         /* high sh */
1598         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1599         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1600         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1601         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
1602         /* low mh */
1603         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1604         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1605         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1606         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1607         /* mid mh */
1608         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1609         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1610         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1611         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
1612         /* high mh */
1613         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1614         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1615         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1616         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
1617 }
1618
1619 /**
1620  * evergreen_pm_misc - set additional pm hw parameters callback.
1621  *
1622  * @rdev: radeon_device pointer
1623  *
1624  * Set non-clock parameters associated with a power state
1625  * (voltage, etc.) (evergreen+).
1626  */
1627 void evergreen_pm_misc(struct radeon_device *rdev)
1628 {
1629         int req_ps_idx = rdev->pm.requested_power_state_index;
1630         int req_cm_idx = rdev->pm.requested_clock_mode_index;
1631         struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
1632         struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
1633
1634         if (voltage->type == VOLTAGE_SW) {
1635                 /* 0xff0x are flags rather then an actual voltage */
1636                 if ((voltage->voltage & 0xff00) == 0xff00)
1637                         return;
1638                 if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
1639                         radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
1640                         rdev->pm.current_vddc = voltage->voltage;
1641                         DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
1642                 }
1643
1644                 /* starting with BTC, there is one state that is used for both
1645                  * MH and SH.  Difference is that we always use the high clock index for
1646                  * mclk and vddci.
1647                  */
1648                 if ((rdev->pm.pm_method == PM_METHOD_PROFILE) &&
1649                     (rdev->family >= CHIP_BARTS) &&
1650                     rdev->pm.active_crtc_count &&
1651                     ((rdev->pm.profile_index == PM_PROFILE_MID_MH_IDX) ||
1652                      (rdev->pm.profile_index == PM_PROFILE_LOW_MH_IDX)))
1653                         voltage = &rdev->pm.power_state[req_ps_idx].
1654                                 clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage;
1655
1656                 /* 0xff0x are flags rather then an actual voltage */
1657                 if ((voltage->vddci & 0xff00) == 0xff00)
1658                         return;
1659                 if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
1660                         radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
1661                         rdev->pm.current_vddci = voltage->vddci;
1662                         DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
1663                 }
1664         }
1665 }
1666
1667 /**
1668  * evergreen_pm_prepare - pre-power state change callback.
1669  *
1670  * @rdev: radeon_device pointer
1671  *
1672  * Prepare for a power state change (evergreen+).
1673  */
1674 void evergreen_pm_prepare(struct radeon_device *rdev)
1675 {
1676         struct drm_device *ddev = rdev->ddev;
1677         struct drm_crtc *crtc;
1678         struct radeon_crtc *radeon_crtc;
1679         u32 tmp;
1680
1681         /* disable any active CRTCs */
1682         list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1683                 radeon_crtc = to_radeon_crtc(crtc);
1684                 if (radeon_crtc->enabled) {
1685                         tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1686                         tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1687                         WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1688                 }
1689         }
1690 }
1691
1692 /**
1693  * evergreen_pm_finish - post-power state change callback.
1694  *
1695  * @rdev: radeon_device pointer
1696  *
1697  * Clean up after a power state change (evergreen+).
1698  */
1699 void evergreen_pm_finish(struct radeon_device *rdev)
1700 {
1701         struct drm_device *ddev = rdev->ddev;
1702         struct drm_crtc *crtc;
1703         struct radeon_crtc *radeon_crtc;
1704         u32 tmp;
1705
1706         /* enable any active CRTCs */
1707         list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1708                 radeon_crtc = to_radeon_crtc(crtc);
1709                 if (radeon_crtc->enabled) {
1710                         tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1711                         tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1712                         WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1713                 }
1714         }
1715 }
1716
1717 /**
1718  * evergreen_hpd_sense - hpd sense callback.
1719  *
1720  * @rdev: radeon_device pointer
1721  * @hpd: hpd (hotplug detect) pin
1722  *
1723  * Checks if a digital monitor is connected (evergreen+).
1724  * Returns true if connected, false if not connected.
1725  */
1726 bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
1727 {
1728         if (hpd == RADEON_HPD_NONE)
1729                 return false;
1730
1731         return !!(RREG32(DC_HPDx_INT_STATUS_REG(hpd)) & DC_HPDx_SENSE);
1732 }
1733
1734 /**
1735  * evergreen_hpd_set_polarity - hpd set polarity callback.
1736  *
1737  * @rdev: radeon_device pointer
1738  * @hpd: hpd (hotplug detect) pin
1739  *
1740  * Set the polarity of the hpd pin (evergreen+).
1741  */
1742 void evergreen_hpd_set_polarity(struct radeon_device *rdev,
1743                                 enum radeon_hpd_id hpd)
1744 {
1745         bool connected = evergreen_hpd_sense(rdev, hpd);
1746
1747         if (hpd == RADEON_HPD_NONE)
1748                 return;
1749
1750         if (connected)
1751                 WREG32_AND(DC_HPDx_INT_CONTROL(hpd), ~DC_HPDx_INT_POLARITY);
1752         else
1753                 WREG32_OR(DC_HPDx_INT_CONTROL(hpd), DC_HPDx_INT_POLARITY);
1754 }
1755
1756 /**
1757  * evergreen_hpd_init - hpd setup callback.
1758  *
1759  * @rdev: radeon_device pointer
1760  *
1761  * Setup the hpd pins used by the card (evergreen+).
1762  * Enable the pin, set the polarity, and enable the hpd interrupts.
1763  */
1764 void evergreen_hpd_init(struct radeon_device *rdev)
1765 {
1766         struct drm_device *dev = rdev->ddev;
1767         struct drm_connector *connector;
1768         unsigned enabled = 0;
1769         u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) |
1770                 DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN;
1771
1772         list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1773                 enum radeon_hpd_id hpd =
1774                         to_radeon_connector(connector)->hpd.hpd;
1775
1776                 if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
1777                     connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
1778                         /* don't try to enable hpd on eDP or LVDS avoid breaking the
1779                          * aux dp channel on imac and help (but not completely fix)
1780                          * https://bugzilla.redhat.com/show_bug.cgi?id=726143
1781                          * also avoid interrupt storms during dpms.
1782                          */
1783                         continue;
1784                 }
1785
1786                 if (hpd == RADEON_HPD_NONE)
1787                         continue;
1788
1789                 WREG32(DC_HPDx_CONTROL(hpd), tmp);
1790                 enabled |= 1 << hpd;
1791
1792                 radeon_hpd_set_polarity(rdev, hpd);
1793         }
1794         radeon_irq_kms_enable_hpd(rdev, enabled);
1795 }
1796
1797 /**
1798  * evergreen_hpd_fini - hpd tear down callback.
1799  *
1800  * @rdev: radeon_device pointer
1801  *
1802  * Tear down the hpd pins used by the card (evergreen+).
1803  * Disable the hpd interrupts.
1804  */
1805 void evergreen_hpd_fini(struct radeon_device *rdev)
1806 {
1807         struct drm_device *dev = rdev->ddev;
1808         struct drm_connector *connector;
1809         unsigned disabled = 0;
1810
1811         list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1812                 enum radeon_hpd_id hpd =
1813                         to_radeon_connector(connector)->hpd.hpd;
1814
1815                 if (hpd == RADEON_HPD_NONE)
1816                         continue;
1817
1818                 WREG32(DC_HPDx_CONTROL(hpd), 0);
1819                 disabled |= 1 << hpd;
1820         }
1821         radeon_irq_kms_disable_hpd(rdev, disabled);
1822 }
1823
1824 /* watermark setup */
1825
1826 static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
1827                                         struct radeon_crtc *radeon_crtc,
1828                                         struct drm_display_mode *mode,
1829                                         struct drm_display_mode *other_mode)
1830 {
1831         u32 tmp, buffer_alloc, i;
1832         u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
1833         /*
1834          * Line Buffer Setup
1835          * There are 3 line buffers, each one shared by 2 display controllers.
1836          * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1837          * the display controllers.  The paritioning is done via one of four
1838          * preset allocations specified in bits 2:0:
1839          * first display controller
1840          *  0 - first half of lb (3840 * 2)
1841          *  1 - first 3/4 of lb (5760 * 2)
1842          *  2 - whole lb (7680 * 2), other crtc must be disabled
1843          *  3 - first 1/4 of lb (1920 * 2)
1844          * second display controller
1845          *  4 - second half of lb (3840 * 2)
1846          *  5 - second 3/4 of lb (5760 * 2)
1847          *  6 - whole lb (7680 * 2), other crtc must be disabled
1848          *  7 - last 1/4 of lb (1920 * 2)
1849          */
1850         /* this can get tricky if we have two large displays on a paired group
1851          * of crtcs.  Ideally for multiple large displays we'd assign them to
1852          * non-linked crtcs for maximum line buffer allocation.
1853          */
1854         if (radeon_crtc->base.enabled && mode) {
1855                 if (other_mode) {
1856                         tmp = 0; /* 1/2 */
1857                         buffer_alloc = 1;
1858                 } else {
1859                         tmp = 2; /* whole */
1860                         buffer_alloc = 2;
1861                 }
1862         } else {
1863                 tmp = 0;
1864                 buffer_alloc = 0;
1865         }
1866
1867         /* second controller of the pair uses second half of the lb */
1868         if (radeon_crtc->crtc_id % 2)
1869                 tmp += 4;
1870         WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
1871
1872         if (ASIC_IS_DCE41(rdev) || ASIC_IS_DCE5(rdev)) {
1873                 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1874                        DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1875                 for (i = 0; i < rdev->usec_timeout; i++) {
1876                         if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1877                             DMIF_BUFFERS_ALLOCATED_COMPLETED)
1878                                 break;
1879                         udelay(1);
1880                 }
1881         }
1882
1883         if (radeon_crtc->base.enabled && mode) {
1884                 switch (tmp) {
1885                 case 0:
1886                 case 4:
1887                 default:
1888                         if (ASIC_IS_DCE5(rdev))
1889                                 return 4096 * 2;
1890                         else
1891                                 return 3840 * 2;
1892                 case 1:
1893                 case 5:
1894                         if (ASIC_IS_DCE5(rdev))
1895                                 return 6144 * 2;
1896                         else
1897                                 return 5760 * 2;
1898                 case 2:
1899                 case 6:
1900                         if (ASIC_IS_DCE5(rdev))
1901                                 return 8192 * 2;
1902                         else
1903                                 return 7680 * 2;
1904                 case 3:
1905                 case 7:
1906                         if (ASIC_IS_DCE5(rdev))
1907                                 return 2048 * 2;
1908                         else
1909                                 return 1920 * 2;
1910                 }
1911         }
1912
1913         /* controller not enabled, so no lb used */
1914         return 0;
1915 }
1916
1917 u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
1918 {
1919         u32 tmp = RREG32(MC_SHARED_CHMAP);
1920
1921         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1922         case 0:
1923         default:
1924                 return 1;
1925         case 1:
1926                 return 2;
1927         case 2:
1928                 return 4;
1929         case 3:
1930                 return 8;
1931         }
1932 }
1933
1934 struct evergreen_wm_params {
1935         u32 dram_channels; /* number of dram channels */
1936         u32 yclk;          /* bandwidth per dram data pin in kHz */
1937         u32 sclk;          /* engine clock in kHz */
1938         u32 disp_clk;      /* display clock in kHz */
1939         u32 src_width;     /* viewport width */
1940         u32 active_time;   /* active display time in ns */
1941         u32 blank_time;    /* blank time in ns */
1942         bool interlaced;    /* mode is interlaced */
1943         fixed20_12 vsc;    /* vertical scale ratio */
1944         u32 num_heads;     /* number of active crtcs */
1945         u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1946         u32 lb_size;       /* line buffer allocated to pipe */
1947         u32 vtaps;         /* vertical scaler taps */
1948 };
1949
1950 static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
1951 {
1952         /* Calculate DRAM Bandwidth and the part allocated to display. */
1953         fixed20_12 dram_efficiency; /* 0.7 */
1954         fixed20_12 yclk, dram_channels, bandwidth;
1955         fixed20_12 a;
1956
1957         a.full = dfixed_const(1000);
1958         yclk.full = dfixed_const(wm->yclk);
1959         yclk.full = dfixed_div(yclk, a);
1960         dram_channels.full = dfixed_const(wm->dram_channels * 4);
1961         a.full = dfixed_const(10);
1962         dram_efficiency.full = dfixed_const(7);
1963         dram_efficiency.full = dfixed_div(dram_efficiency, a);
1964         bandwidth.full = dfixed_mul(dram_channels, yclk);
1965         bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1966
1967         return dfixed_trunc(bandwidth);
1968 }
1969
1970 static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
1971 {
1972         /* Calculate DRAM Bandwidth and the part allocated to display. */
1973         fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
1974         fixed20_12 yclk, dram_channels, bandwidth;
1975         fixed20_12 a;
1976
1977         a.full = dfixed_const(1000);
1978         yclk.full = dfixed_const(wm->yclk);
1979         yclk.full = dfixed_div(yclk, a);
1980         dram_channels.full = dfixed_const(wm->dram_channels * 4);
1981         a.full = dfixed_const(10);
1982         disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
1983         disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
1984         bandwidth.full = dfixed_mul(dram_channels, yclk);
1985         bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
1986
1987         return dfixed_trunc(bandwidth);
1988 }
1989
1990 static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
1991 {
1992         /* Calculate the display Data return Bandwidth */
1993         fixed20_12 return_efficiency; /* 0.8 */
1994         fixed20_12 sclk, bandwidth;
1995         fixed20_12 a;
1996
1997         a.full = dfixed_const(1000);
1998         sclk.full = dfixed_const(wm->sclk);
1999         sclk.full = dfixed_div(sclk, a);
2000         a.full = dfixed_const(10);
2001         return_efficiency.full = dfixed_const(8);
2002         return_efficiency.full = dfixed_div(return_efficiency, a);
2003         a.full = dfixed_const(32);
2004         bandwidth.full = dfixed_mul(a, sclk);
2005         bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
2006
2007         return dfixed_trunc(bandwidth);
2008 }
2009
2010 static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
2011 {
2012         /* Calculate the DMIF Request Bandwidth */
2013         fixed20_12 disp_clk_request_efficiency; /* 0.8 */
2014         fixed20_12 disp_clk, bandwidth;
2015         fixed20_12 a;
2016
2017         a.full = dfixed_const(1000);
2018         disp_clk.full = dfixed_const(wm->disp_clk);
2019         disp_clk.full = dfixed_div(disp_clk, a);
2020         a.full = dfixed_const(10);
2021         disp_clk_request_efficiency.full = dfixed_const(8);
2022         disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
2023         a.full = dfixed_const(32);
2024         bandwidth.full = dfixed_mul(a, disp_clk);
2025         bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
2026
2027         return dfixed_trunc(bandwidth);
2028 }
2029
2030 static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
2031 {
2032         /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
2033         u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
2034         u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
2035         u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
2036
2037         return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
2038 }
2039
2040 static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
2041 {
2042         /* Calculate the display mode Average Bandwidth
2043          * DisplayMode should contain the source and destination dimensions,
2044          * timing, etc.
2045          */
2046         fixed20_12 bpp;
2047         fixed20_12 line_time;
2048         fixed20_12 src_width;
2049         fixed20_12 bandwidth;
2050         fixed20_12 a;
2051
2052         a.full = dfixed_const(1000);
2053         line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2054         line_time.full = dfixed_div(line_time, a);
2055         bpp.full = dfixed_const(wm->bytes_per_pixel);
2056         src_width.full = dfixed_const(wm->src_width);
2057         bandwidth.full = dfixed_mul(src_width, bpp);
2058         bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2059         bandwidth.full = dfixed_div(bandwidth, line_time);
2060
2061         return dfixed_trunc(bandwidth);
2062 }
2063
2064 static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
2065 {
2066         /* First calcualte the latency in ns */
2067         u32 mc_latency = 2000; /* 2000 ns. */
2068         u32 available_bandwidth = evergreen_available_bandwidth(wm);
2069         u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2070         u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2071         u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2072         u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2073                 (wm->num_heads * cursor_line_pair_return_time);
2074         u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2075         u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2076         fixed20_12 a, b, c;
2077
2078         if (wm->num_heads == 0)
2079                 return 0;
2080
2081         a.full = dfixed_const(2);
2082         b.full = dfixed_const(1);
2083         if ((wm->vsc.full > a.full) ||
2084             ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2085             (wm->vtaps >= 5) ||
2086             ((wm->vsc.full >= a.full) && wm->interlaced))
2087                 max_src_lines_per_dst_line = 4;
2088         else
2089                 max_src_lines_per_dst_line = 2;
2090
2091         a.full = dfixed_const(available_bandwidth);
2092         b.full = dfixed_const(wm->num_heads);
2093         a.full = dfixed_div(a, b);
2094
2095         lb_fill_bw = min(dfixed_trunc(a), wm->disp_clk * wm->bytes_per_pixel / 1000);
2096
2097         a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2098         b.full = dfixed_const(1000);
2099         c.full = dfixed_const(lb_fill_bw);
2100         b.full = dfixed_div(c, b);
2101         a.full = dfixed_div(a, b);
2102         line_fill_time = dfixed_trunc(a);
2103
2104         if (line_fill_time < wm->active_time)
2105                 return latency;
2106         else
2107                 return latency + (line_fill_time - wm->active_time);
2108
2109 }
2110
2111 static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2112 {
2113         if (evergreen_average_bandwidth(wm) <=
2114             (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
2115                 return true;
2116         else
2117                 return false;
2118 };
2119
2120 static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
2121 {
2122         if (evergreen_average_bandwidth(wm) <=
2123             (evergreen_available_bandwidth(wm) / wm->num_heads))
2124                 return true;
2125         else
2126                 return false;
2127 };
2128
2129 static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
2130 {
2131         u32 lb_partitions = wm->lb_size / wm->src_width;
2132         u32 line_time = wm->active_time + wm->blank_time;
2133         u32 latency_tolerant_lines;
2134         u32 latency_hiding;
2135         fixed20_12 a;
2136
2137         a.full = dfixed_const(1);
2138         if (wm->vsc.full > a.full)
2139                 latency_tolerant_lines = 1;
2140         else {
2141                 if (lb_partitions <= (wm->vtaps + 1))
2142                         latency_tolerant_lines = 1;
2143                 else
2144                         latency_tolerant_lines = 2;
2145         }
2146
2147         latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2148
2149         if (evergreen_latency_watermark(wm) <= latency_hiding)
2150                 return true;
2151         else
2152                 return false;
2153 }
2154
2155 static void evergreen_program_watermarks(struct radeon_device *rdev,
2156                                          struct radeon_crtc *radeon_crtc,
2157                                          u32 lb_size, u32 num_heads)
2158 {
2159         struct drm_display_mode *mode = &radeon_crtc->base.mode;
2160         struct evergreen_wm_params wm_low, wm_high;
2161         u32 dram_channels;
2162         u32 active_time;
2163         u32 line_time = 0;
2164         u32 latency_watermark_a = 0, latency_watermark_b = 0;
2165         u32 priority_a_mark = 0, priority_b_mark = 0;
2166         u32 priority_a_cnt = PRIORITY_OFF;
2167         u32 priority_b_cnt = PRIORITY_OFF;
2168         u32 pipe_offset = radeon_crtc->crtc_id * 16;
2169         u32 tmp, arb_control3;
2170         fixed20_12 a, b, c;
2171
2172         if (radeon_crtc->base.enabled && num_heads && mode) {
2173                 active_time = (u32) div_u64((u64)mode->crtc_hdisplay * 1000000,
2174                                             (u32)mode->clock);
2175                 line_time = (u32) div_u64((u64)mode->crtc_htotal * 1000000,
2176                                           (u32)mode->clock);
2177                 line_time = min(line_time, (u32)65535);
2178                 priority_a_cnt = 0;
2179                 priority_b_cnt = 0;
2180                 dram_channels = evergreen_get_number_of_dram_channels(rdev);
2181
2182                 /* watermark for high clocks */
2183                 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2184                         wm_high.yclk =
2185                                 radeon_dpm_get_mclk(rdev, false) * 10;
2186                         wm_high.sclk =
2187                                 radeon_dpm_get_sclk(rdev, false) * 10;
2188                 } else {
2189                         wm_high.yclk = rdev->pm.current_mclk * 10;
2190                         wm_high.sclk = rdev->pm.current_sclk * 10;
2191                 }
2192
2193                 wm_high.disp_clk = mode->clock;
2194                 wm_high.src_width = mode->crtc_hdisplay;
2195                 wm_high.active_time = active_time;
2196                 wm_high.blank_time = line_time - wm_high.active_time;
2197                 wm_high.interlaced = false;
2198                 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2199                         wm_high.interlaced = true;
2200                 wm_high.vsc = radeon_crtc->vsc;
2201                 wm_high.vtaps = 1;
2202                 if (radeon_crtc->rmx_type != RMX_OFF)
2203                         wm_high.vtaps = 2;
2204                 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2205                 wm_high.lb_size = lb_size;
2206                 wm_high.dram_channels = dram_channels;
2207                 wm_high.num_heads = num_heads;
2208
2209                 /* watermark for low clocks */
2210                 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2211                         wm_low.yclk =
2212                                 radeon_dpm_get_mclk(rdev, true) * 10;
2213                         wm_low.sclk =
2214                                 radeon_dpm_get_sclk(rdev, true) * 10;
2215                 } else {
2216                         wm_low.yclk = rdev->pm.current_mclk * 10;
2217                         wm_low.sclk = rdev->pm.current_sclk * 10;
2218                 }
2219
2220                 wm_low.disp_clk = mode->clock;
2221                 wm_low.src_width = mode->crtc_hdisplay;
2222                 wm_low.active_time = active_time;
2223                 wm_low.blank_time = line_time - wm_low.active_time;
2224                 wm_low.interlaced = false;
2225                 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2226                         wm_low.interlaced = true;
2227                 wm_low.vsc = radeon_crtc->vsc;
2228                 wm_low.vtaps = 1;
2229                 if (radeon_crtc->rmx_type != RMX_OFF)
2230                         wm_low.vtaps = 2;
2231                 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2232                 wm_low.lb_size = lb_size;
2233                 wm_low.dram_channels = dram_channels;
2234                 wm_low.num_heads = num_heads;
2235
2236                 /* set for high clocks */
2237                 latency_watermark_a = min(evergreen_latency_watermark(&wm_high), (u32)65535);
2238                 /* set for low clocks */
2239                 latency_watermark_b = min(evergreen_latency_watermark(&wm_low), (u32)65535);
2240
2241                 /* possibly force display priority to high */
2242                 /* should really do this at mode validation time... */
2243                 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2244                     !evergreen_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2245                     !evergreen_check_latency_hiding(&wm_high) ||
2246                     (rdev->disp_priority == 2)) {
2247                         DRM_DEBUG_KMS("force priority a to high\n");
2248                         priority_a_cnt |= PRIORITY_ALWAYS_ON;
2249                 }
2250                 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2251                     !evergreen_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2252                     !evergreen_check_latency_hiding(&wm_low) ||
2253                     (rdev->disp_priority == 2)) {
2254                         DRM_DEBUG_KMS("force priority b to high\n");
2255                         priority_b_cnt |= PRIORITY_ALWAYS_ON;
2256                 }
2257
2258                 a.full = dfixed_const(1000);
2259                 b.full = dfixed_const(mode->clock);
2260                 b.full = dfixed_div(b, a);
2261                 c.full = dfixed_const(latency_watermark_a);
2262                 c.full = dfixed_mul(c, b);
2263                 c.full = dfixed_mul(c, radeon_crtc->hsc);
2264                 c.full = dfixed_div(c, a);
2265                 a.full = dfixed_const(16);
2266                 c.full = dfixed_div(c, a);
2267                 priority_a_mark = dfixed_trunc(c);
2268                 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2269
2270                 a.full = dfixed_const(1000);
2271                 b.full = dfixed_const(mode->clock);
2272                 b.full = dfixed_div(b, a);
2273                 c.full = dfixed_const(latency_watermark_b);
2274                 c.full = dfixed_mul(c, b);
2275                 c.full = dfixed_mul(c, radeon_crtc->hsc);
2276                 c.full = dfixed_div(c, a);
2277                 a.full = dfixed_const(16);
2278                 c.full = dfixed_div(c, a);
2279                 priority_b_mark = dfixed_trunc(c);
2280                 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2281
2282                 /* Save number of lines the linebuffer leads before the scanout */
2283                 radeon_crtc->lb_vblank_lead_lines = DIV_ROUND_UP(lb_size, mode->crtc_hdisplay);
2284         }
2285
2286         /* select wm A */
2287         arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2288         tmp = arb_control3;
2289         tmp &= ~LATENCY_WATERMARK_MASK(3);
2290         tmp |= LATENCY_WATERMARK_MASK(1);
2291         WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2292         WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2293                (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2294                 LATENCY_HIGH_WATERMARK(line_time)));
2295         /* select wm B */
2296         tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2297         tmp &= ~LATENCY_WATERMARK_MASK(3);
2298         tmp |= LATENCY_WATERMARK_MASK(2);
2299         WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2300         WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2301                (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2302                 LATENCY_HIGH_WATERMARK(line_time)));
2303         /* restore original selection */
2304         WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
2305
2306         /* write the priority marks */
2307         WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2308         WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2309
2310         /* save values for DPM */
2311         radeon_crtc->line_time = line_time;
2312         radeon_crtc->wm_high = latency_watermark_a;
2313         radeon_crtc->wm_low = latency_watermark_b;
2314 }
2315
2316 /**
2317  * evergreen_bandwidth_update - update display watermarks callback.
2318  *
2319  * @rdev: radeon_device pointer
2320  *
2321  * Update the display watermarks based on the requested mode(s)
2322  * (evergreen+).
2323  */
2324 void evergreen_bandwidth_update(struct radeon_device *rdev)
2325 {
2326         struct drm_display_mode *mode0 = NULL;
2327         struct drm_display_mode *mode1 = NULL;
2328         u32 num_heads = 0, lb_size;
2329         int i;
2330
2331         if (!rdev->mode_info.mode_config_initialized)
2332                 return;
2333
2334         radeon_update_display_priority(rdev);
2335
2336         for (i = 0; i < rdev->num_crtc; i++) {
2337                 if (rdev->mode_info.crtcs[i]->base.enabled)
2338                         num_heads++;
2339         }
2340         for (i = 0; i < rdev->num_crtc; i += 2) {
2341                 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2342                 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2343                 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2344                 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2345                 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2346                 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2347         }
2348 }
2349
2350 /**
2351  * evergreen_mc_wait_for_idle - wait for MC idle callback.
2352  *
2353  * @rdev: radeon_device pointer
2354  *
2355  * Wait for the MC (memory controller) to be idle.
2356  * (evergreen+).
2357  * Returns 0 if the MC is idle, -1 if not.
2358  */
2359 int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
2360 {
2361         unsigned i;
2362         u32 tmp;
2363
2364         for (i = 0; i < rdev->usec_timeout; i++) {
2365                 /* read MC_STATUS */
2366                 tmp = RREG32(SRBM_STATUS) & 0x1F00;
2367                 if (!tmp)
2368                         return 0;
2369                 udelay(1);
2370         }
2371         return -1;
2372 }
2373
2374 /*
2375  * GART
2376  */
2377 void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
2378 {
2379         unsigned i;
2380         u32 tmp;
2381
2382         WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
2383
2384         WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
2385         for (i = 0; i < rdev->usec_timeout; i++) {
2386                 /* read MC_STATUS */
2387                 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
2388                 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
2389                 if (tmp == 2) {
2390                         pr_warn("[drm] r600 flush TLB failed\n");
2391                         return;
2392                 }
2393                 if (tmp) {
2394                         return;
2395                 }
2396                 udelay(1);
2397         }
2398 }
2399
2400 static int evergreen_pcie_gart_enable(struct radeon_device *rdev)
2401 {
2402         u32 tmp;
2403         int r;
2404
2405         if (rdev->gart.robj == NULL) {
2406                 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
2407                 return -EINVAL;
2408         }
2409         r = radeon_gart_table_vram_pin(rdev);
2410         if (r)
2411                 return r;
2412         /* Setup L2 cache */
2413         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2414                                 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2415                                 EFFECTIVE_L2_QUEUE_SIZE(7));
2416         WREG32(VM_L2_CNTL2, 0);
2417         WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2418         /* Setup TLB control */
2419         tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2420                 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2421                 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2422                 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2423         if (rdev->flags & RADEON_IS_IGP) {
2424                 WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
2425                 WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
2426                 WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
2427         } else {
2428                 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2429                 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2430                 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2431                 if ((rdev->family == CHIP_JUNIPER) ||
2432                     (rdev->family == CHIP_CYPRESS) ||
2433                     (rdev->family == CHIP_HEMLOCK) ||
2434                     (rdev->family == CHIP_BARTS))
2435                         WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
2436         }
2437         WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2438         WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2439         WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2440         WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2441         WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
2442         WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
2443         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
2444         WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
2445                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
2446         WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
2447                         (u32)(rdev->dummy_page.addr >> 12));
2448         WREG32(VM_CONTEXT1_CNTL, 0);
2449
2450         evergreen_pcie_gart_tlb_flush(rdev);
2451         DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
2452                  (unsigned)(rdev->mc.gtt_size >> 20),
2453                  (unsigned long long)rdev->gart.table_addr);
2454         rdev->gart.ready = true;
2455         return 0;
2456 }
2457
2458 static void evergreen_pcie_gart_disable(struct radeon_device *rdev)
2459 {
2460         u32 tmp;
2461
2462         /* Disable all tables */
2463         WREG32(VM_CONTEXT0_CNTL, 0);
2464         WREG32(VM_CONTEXT1_CNTL, 0);
2465
2466         /* Setup L2 cache */
2467         WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
2468                                 EFFECTIVE_L2_QUEUE_SIZE(7));
2469         WREG32(VM_L2_CNTL2, 0);
2470         WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2471         /* Setup TLB control */
2472         tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2473         WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2474         WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2475         WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2476         WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2477         WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2478         WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2479         WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2480         radeon_gart_table_vram_unpin(rdev);
2481 }
2482
2483 static void evergreen_pcie_gart_fini(struct radeon_device *rdev)
2484 {
2485         evergreen_pcie_gart_disable(rdev);
2486         radeon_gart_table_vram_free(rdev);
2487         radeon_gart_fini(rdev);
2488 }
2489
2490
2491 static void evergreen_agp_enable(struct radeon_device *rdev)
2492 {
2493         u32 tmp;
2494
2495         /* Setup L2 cache */
2496         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2497                                 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2498                                 EFFECTIVE_L2_QUEUE_SIZE(7));
2499         WREG32(VM_L2_CNTL2, 0);
2500         WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2501         /* Setup TLB control */
2502         tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2503                 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2504                 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2505                 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2506         WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2507         WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2508         WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2509         WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2510         WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2511         WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2512         WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2513         WREG32(VM_CONTEXT0_CNTL, 0);
2514         WREG32(VM_CONTEXT1_CNTL, 0);
2515 }
2516
2517 static const unsigned ni_dig_offsets[] =
2518 {
2519         NI_DIG0_REGISTER_OFFSET,
2520         NI_DIG1_REGISTER_OFFSET,
2521         NI_DIG2_REGISTER_OFFSET,
2522         NI_DIG3_REGISTER_OFFSET,
2523         NI_DIG4_REGISTER_OFFSET,
2524         NI_DIG5_REGISTER_OFFSET
2525 };
2526
2527 static const unsigned ni_tx_offsets[] =
2528 {
2529         NI_DCIO_UNIPHY0_UNIPHY_TX_CONTROL1,
2530         NI_DCIO_UNIPHY1_UNIPHY_TX_CONTROL1,
2531         NI_DCIO_UNIPHY2_UNIPHY_TX_CONTROL1,
2532         NI_DCIO_UNIPHY3_UNIPHY_TX_CONTROL1,
2533         NI_DCIO_UNIPHY4_UNIPHY_TX_CONTROL1,
2534         NI_DCIO_UNIPHY5_UNIPHY_TX_CONTROL1
2535 };
2536
2537 static const unsigned evergreen_dp_offsets[] =
2538 {
2539         EVERGREEN_DP0_REGISTER_OFFSET,
2540         EVERGREEN_DP1_REGISTER_OFFSET,
2541         EVERGREEN_DP2_REGISTER_OFFSET,
2542         EVERGREEN_DP3_REGISTER_OFFSET,
2543         EVERGREEN_DP4_REGISTER_OFFSET,
2544         EVERGREEN_DP5_REGISTER_OFFSET
2545 };
2546
2547 static const unsigned evergreen_disp_int_status[] =
2548 {
2549         DISP_INTERRUPT_STATUS,
2550         DISP_INTERRUPT_STATUS_CONTINUE,
2551         DISP_INTERRUPT_STATUS_CONTINUE2,
2552         DISP_INTERRUPT_STATUS_CONTINUE3,
2553         DISP_INTERRUPT_STATUS_CONTINUE4,
2554         DISP_INTERRUPT_STATUS_CONTINUE5
2555 };
2556
2557 /*
2558  * Assumption is that EVERGREEN_CRTC_MASTER_EN enable for requested crtc
2559  * We go from crtc to connector and it is not relible  since it
2560  * should be an opposite direction .If crtc is enable then
2561  * find the dig_fe which selects this crtc and insure that it enable.
2562  * if such dig_fe is found then find dig_be which selects found dig_be and
2563  * insure that it enable and in DP_SST mode.
2564  * if UNIPHY_PLL_CONTROL1.enable then we should disconnect timing
2565  * from dp symbols clocks .
2566  */
2567 static bool evergreen_is_dp_sst_stream_enabled(struct radeon_device *rdev,
2568                                                unsigned crtc_id, unsigned *ret_dig_fe)
2569 {
2570         unsigned i;
2571         unsigned dig_fe;
2572         unsigned dig_be;
2573         unsigned dig_en_be;
2574         unsigned uniphy_pll;
2575         unsigned digs_fe_selected;
2576         unsigned dig_be_mode;
2577         unsigned dig_fe_mask;
2578         bool is_enabled = false;
2579         bool found_crtc = false;
2580
2581         /* loop through all running dig_fe to find selected crtc */
2582         for (i = 0; i < ARRAY_SIZE(ni_dig_offsets); i++) {
2583                 dig_fe = RREG32(NI_DIG_FE_CNTL + ni_dig_offsets[i]);
2584                 if (dig_fe & NI_DIG_FE_CNTL_SYMCLK_FE_ON &&
2585                     crtc_id == NI_DIG_FE_CNTL_SOURCE_SELECT(dig_fe)) {
2586                         /* found running pipe */
2587                         found_crtc = true;
2588                         dig_fe_mask = 1 << i;
2589                         dig_fe = i;
2590                         break;
2591                 }
2592         }
2593
2594         if (found_crtc) {
2595                 /* loop through all running dig_be to find selected dig_fe */
2596                 for (i = 0; i < ARRAY_SIZE(ni_dig_offsets); i++) {
2597                         dig_be = RREG32(NI_DIG_BE_CNTL + ni_dig_offsets[i]);
2598                         /* if dig_fe_selected by dig_be? */
2599                         digs_fe_selected = NI_DIG_BE_CNTL_FE_SOURCE_SELECT(dig_be);
2600                         dig_be_mode = NI_DIG_FE_CNTL_MODE(dig_be);
2601                         if (dig_fe_mask &  digs_fe_selected &&
2602                             /* if dig_be in sst mode? */
2603                             dig_be_mode == NI_DIG_BE_DPSST) {
2604                                 dig_en_be = RREG32(NI_DIG_BE_EN_CNTL +
2605                                                    ni_dig_offsets[i]);
2606                                 uniphy_pll = RREG32(NI_DCIO_UNIPHY0_PLL_CONTROL1 +
2607                                                     ni_tx_offsets[i]);
2608                                 /* dig_be enable and tx is running */
2609                                 if (dig_en_be & NI_DIG_BE_EN_CNTL_ENABLE &&
2610                                     dig_en_be & NI_DIG_BE_EN_CNTL_SYMBCLK_ON &&
2611                                     uniphy_pll & NI_DCIO_UNIPHY0_PLL_CONTROL1_ENABLE) {
2612                                         is_enabled = true;
2613                                         *ret_dig_fe = dig_fe;
2614                                         break;
2615                                 }
2616                         }
2617                 }
2618         }
2619
2620         return is_enabled;
2621 }
2622
2623 /*
2624  * Blank dig when in dp sst mode
2625  * Dig ignores crtc timing
2626  */
2627 static void evergreen_blank_dp_output(struct radeon_device *rdev,
2628                                       unsigned dig_fe)
2629 {
2630         unsigned stream_ctrl;
2631         unsigned fifo_ctrl;
2632         unsigned counter = 0;
2633
2634         if (dig_fe >= ARRAY_SIZE(evergreen_dp_offsets)) {
2635                 DRM_ERROR("invalid dig_fe %d\n", dig_fe);
2636                 return;
2637         }
2638
2639         stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2640                              evergreen_dp_offsets[dig_fe]);
2641         if (!(stream_ctrl & EVERGREEN_DP_VID_STREAM_CNTL_ENABLE)) {
2642                 DRM_ERROR("dig %d , should be enable\n", dig_fe);
2643                 return;
2644         }
2645
2646         stream_ctrl &=~EVERGREEN_DP_VID_STREAM_CNTL_ENABLE;
2647         WREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2648                evergreen_dp_offsets[dig_fe], stream_ctrl);
2649
2650         stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2651                              evergreen_dp_offsets[dig_fe]);
2652         while (counter < 32 && stream_ctrl & EVERGREEN_DP_VID_STREAM_STATUS) {
2653                 msleep(1);
2654                 counter++;
2655                 stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2656                                      evergreen_dp_offsets[dig_fe]);
2657         }
2658         if (counter >= 32 )
2659                 DRM_ERROR("counter exceeds %d\n", counter);
2660
2661         fifo_ctrl = RREG32(EVERGREEN_DP_STEER_FIFO + evergreen_dp_offsets[dig_fe]);
2662         fifo_ctrl |= EVERGREEN_DP_STEER_FIFO_RESET;
2663         WREG32(EVERGREEN_DP_STEER_FIFO + evergreen_dp_offsets[dig_fe], fifo_ctrl);
2664
2665 }
2666
2667 void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save)
2668 {
2669         u32 crtc_enabled, tmp, frame_count, blackout;
2670         int i, j;
2671         unsigned dig_fe;
2672
2673         if (!ASIC_IS_NODCE(rdev)) {
2674                 save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
2675                 save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
2676
2677                 /* disable VGA render */
2678                 WREG32(VGA_RENDER_CONTROL, 0);
2679         }
2680         /* blank the display controllers */
2681         for (i = 0; i < rdev->num_crtc; i++) {
2682                 crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN;
2683                 if (crtc_enabled) {
2684                         save->crtc_enabled[i] = true;
2685                         if (ASIC_IS_DCE6(rdev)) {
2686                                 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2687                                 if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) {
2688                                         radeon_wait_for_vblank(rdev, i);
2689                                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2690                                         tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
2691                                         WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2692                                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2693                                 }
2694                         } else {
2695                                 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2696                                 if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) {
2697                                         radeon_wait_for_vblank(rdev, i);
2698                                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2699                                         tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2700                                         WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2701                                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2702                                 }
2703                         }
2704                         /* wait for the next frame */
2705                         frame_count = radeon_get_vblank_counter(rdev, i);
2706                         for (j = 0; j < rdev->usec_timeout; j++) {
2707                                 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2708                                         break;
2709                                 udelay(1);
2710                         }
2711                         /*we should disable dig if it drives dp sst*/
2712                         /*but we are in radeon_device_init and the topology is unknown*/
2713                         /*and it is available after radeon_modeset_init*/
2714                         /*the following method radeon_atom_encoder_dpms_dig*/
2715                         /*does the job if we initialize it properly*/
2716                         /*for now we do it this manually*/
2717                         /**/
2718                         if (ASIC_IS_DCE5(rdev) &&
2719                             evergreen_is_dp_sst_stream_enabled(rdev, i ,&dig_fe))
2720                                 evergreen_blank_dp_output(rdev, dig_fe);
2721                         /*we could remove 6 lines below*/
2722                         /* XXX this is a hack to avoid strange behavior with EFI on certain systems */
2723                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2724                         tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2725                         tmp &= ~EVERGREEN_CRTC_MASTER_EN;
2726                         WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2727                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2728                         save->crtc_enabled[i] = false;
2729                         /* ***** */
2730                 } else {
2731                         save->crtc_enabled[i] = false;
2732                 }
2733         }
2734
2735         radeon_mc_wait_for_idle(rdev);
2736
2737         blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
2738         if ((blackout & BLACKOUT_MODE_MASK) != 1) {
2739                 /* Block CPU access */
2740                 WREG32(BIF_FB_EN, 0);
2741                 /* blackout the MC */
2742                 blackout &= ~BLACKOUT_MODE_MASK;
2743                 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
2744         }
2745         /* wait for the MC to settle */
2746         udelay(100);
2747
2748         /* lock double buffered regs */
2749         for (i = 0; i < rdev->num_crtc; i++) {
2750                 if (save->crtc_enabled[i]) {
2751                         tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2752                         if (!(tmp & EVERGREEN_GRPH_UPDATE_LOCK)) {
2753                                 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
2754                                 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2755                         }
2756                         tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2757                         if (!(tmp & 1)) {
2758                                 tmp |= 1;
2759                                 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2760                         }
2761                 }
2762         }
2763 }
2764
2765 void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save)
2766 {
2767         u32 tmp, frame_count;
2768         int i, j;
2769
2770         /* update crtc base addresses */
2771         for (i = 0; i < rdev->num_crtc; i++) {
2772                 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2773                        upper_32_bits(rdev->mc.vram_start));
2774                 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2775                        upper_32_bits(rdev->mc.vram_start));
2776                 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
2777                        (u32)rdev->mc.vram_start);
2778                 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i],
2779                        (u32)rdev->mc.vram_start);
2780         }
2781
2782         if (!ASIC_IS_NODCE(rdev)) {
2783                 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
2784                 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
2785         }
2786
2787         /* unlock regs and wait for update */
2788         for (i = 0; i < rdev->num_crtc; i++) {
2789                 if (save->crtc_enabled[i]) {
2790                         tmp = RREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i]);
2791                         if ((tmp & 0x7) != 0) {
2792                                 tmp &= ~0x7;
2793                                 WREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i], tmp);
2794                         }
2795                         tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2796                         if (tmp & EVERGREEN_GRPH_UPDATE_LOCK) {
2797                                 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
2798                                 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2799                         }
2800                         tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2801                         if (tmp & 1) {
2802                                 tmp &= ~1;
2803                                 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2804                         }
2805                         for (j = 0; j < rdev->usec_timeout; j++) {
2806                                 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2807                                 if ((tmp & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING) == 0)
2808                                         break;
2809                                 udelay(1);
2810                         }
2811                 }
2812         }
2813
2814         /* unblackout the MC */
2815         tmp = RREG32(MC_SHARED_BLACKOUT_CNTL);
2816         tmp &= ~BLACKOUT_MODE_MASK;
2817         WREG32(MC_SHARED_BLACKOUT_CNTL, tmp);
2818         /* allow CPU access */
2819         WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2820
2821         for (i = 0; i < rdev->num_crtc; i++) {
2822                 if (save->crtc_enabled[i]) {
2823                         if (ASIC_IS_DCE6(rdev)) {
2824                                 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2825                                 tmp &= ~EVERGREEN_CRTC_BLANK_DATA_EN;
2826                                 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2827                                 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2828                                 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2829                         } else {
2830                                 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2831                                 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2832                                 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2833                                 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2834                                 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2835                         }
2836                         /* wait for the next frame */
2837                         frame_count = radeon_get_vblank_counter(rdev, i);
2838                         for (j = 0; j < rdev->usec_timeout; j++) {
2839                                 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2840                                         break;
2841                                 udelay(1);
2842                         }
2843                 }
2844         }
2845         if (!ASIC_IS_NODCE(rdev)) {
2846                 /* Unlock vga access */
2847                 WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
2848                 mdelay(1);
2849                 WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
2850         }
2851 }
2852
2853 void evergreen_mc_program(struct radeon_device *rdev)
2854 {
2855         struct evergreen_mc_save save;
2856         u32 tmp;
2857         int i, j;
2858
2859         /* Initialize HDP */
2860         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2861                 WREG32((0x2c14 + j), 0x00000000);
2862                 WREG32((0x2c18 + j), 0x00000000);
2863                 WREG32((0x2c1c + j), 0x00000000);
2864                 WREG32((0x2c20 + j), 0x00000000);
2865                 WREG32((0x2c24 + j), 0x00000000);
2866         }
2867         WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
2868
2869         evergreen_mc_stop(rdev, &save);
2870         if (evergreen_mc_wait_for_idle(rdev)) {
2871                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2872         }
2873         /* Lockout access through VGA aperture*/
2874         WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
2875         /* Update configuration */
2876         if (rdev->flags & RADEON_IS_AGP) {
2877                 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
2878                         /* VRAM before AGP */
2879                         WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2880                                 rdev->mc.vram_start >> 12);
2881                         WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2882                                 rdev->mc.gtt_end >> 12);
2883                 } else {
2884                         /* VRAM after AGP */
2885                         WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2886                                 rdev->mc.gtt_start >> 12);
2887                         WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2888                                 rdev->mc.vram_end >> 12);
2889                 }
2890         } else {
2891                 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2892                         rdev->mc.vram_start >> 12);
2893                 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2894                         rdev->mc.vram_end >> 12);
2895         }
2896         WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
2897         /* llano/ontario only */
2898         if ((rdev->family == CHIP_PALM) ||
2899             (rdev->family == CHIP_SUMO) ||
2900             (rdev->family == CHIP_SUMO2)) {
2901                 tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
2902                 tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
2903                 tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
2904                 WREG32(MC_FUS_VM_FB_OFFSET, tmp);
2905         }
2906         tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
2907         tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
2908         WREG32(MC_VM_FB_LOCATION, tmp);
2909         WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
2910         WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
2911         WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
2912         if (rdev->flags & RADEON_IS_AGP) {
2913                 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
2914                 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
2915                 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
2916         } else {
2917                 WREG32(MC_VM_AGP_BASE, 0);
2918                 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
2919                 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
2920         }
2921         if (evergreen_mc_wait_for_idle(rdev)) {
2922                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2923         }
2924         evergreen_mc_resume(rdev, &save);
2925         /* we need to own VRAM, so turn off the VGA renderer here
2926          * to stop it overwriting our objects */
2927         rv515_vga_render_disable(rdev);
2928 }
2929
2930 /*
2931  * CP.
2932  */
2933 void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
2934 {
2935         struct radeon_ring *ring = &rdev->ring[ib->ring];
2936         u32 next_rptr;
2937
2938         /* set to DX10/11 mode */
2939         radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
2940         radeon_ring_write(ring, 1);
2941
2942         if (ring->rptr_save_reg) {
2943                 next_rptr = ring->wptr + 3 + 4;
2944                 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2945                 radeon_ring_write(ring, ((ring->rptr_save_reg -
2946                                           PACKET3_SET_CONFIG_REG_START) >> 2));
2947                 radeon_ring_write(ring, next_rptr);
2948         } else if (rdev->wb.enabled) {
2949                 next_rptr = ring->wptr + 5 + 4;
2950                 radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
2951                 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
2952                 radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18));
2953                 radeon_ring_write(ring, next_rptr);
2954                 radeon_ring_write(ring, 0);
2955         }
2956
2957         radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
2958         radeon_ring_write(ring,
2959 #ifdef __BIG_ENDIAN
2960                           (2 << 0) |
2961 #endif
2962                           (ib->gpu_addr & 0xFFFFFFFC));
2963         radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
2964         radeon_ring_write(ring, ib->length_dw);
2965 }
2966
2967
2968 static int evergreen_cp_load_microcode(struct radeon_device *rdev)
2969 {
2970         const __be32 *fw_data;
2971         int i;
2972
2973         if (!rdev->me_fw || !rdev->pfp_fw)
2974                 return -EINVAL;
2975
2976         r700_cp_stop(rdev);
2977         WREG32(CP_RB_CNTL,
2978 #ifdef __BIG_ENDIAN
2979                BUF_SWAP_32BIT |
2980 #endif
2981                RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
2982
2983         fw_data = (const __be32 *)rdev->pfp_fw->data;
2984         WREG32(CP_PFP_UCODE_ADDR, 0);
2985         for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
2986                 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
2987         WREG32(CP_PFP_UCODE_ADDR, 0);
2988
2989         fw_data = (const __be32 *)rdev->me_fw->data;
2990         WREG32(CP_ME_RAM_WADDR, 0);
2991         for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
2992                 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
2993
2994         WREG32(CP_PFP_UCODE_ADDR, 0);
2995         WREG32(CP_ME_RAM_WADDR, 0);
2996         WREG32(CP_ME_RAM_RADDR, 0);
2997         return 0;
2998 }
2999
3000 static int evergreen_cp_start(struct radeon_device *rdev)
3001 {
3002         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3003         int r, i;
3004         uint32_t cp_me;
3005
3006         r = radeon_ring_lock(rdev, ring, 7);
3007         if (r) {
3008                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3009                 return r;
3010         }
3011         radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
3012         radeon_ring_write(ring, 0x1);
3013         radeon_ring_write(ring, 0x0);
3014         radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1);
3015         radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
3016         radeon_ring_write(ring, 0);
3017         radeon_ring_write(ring, 0);
3018         radeon_ring_unlock_commit(rdev, ring, false);
3019
3020         cp_me = 0xff;
3021         WREG32(CP_ME_CNTL, cp_me);
3022
3023         r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19);
3024         if (r) {
3025                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3026                 return r;
3027         }
3028
3029         /* setup clear context state */
3030         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3031         radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3032
3033         for (i = 0; i < evergreen_default_size; i++)
3034                 radeon_ring_write(ring, evergreen_default_state[i]);
3035
3036         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3037         radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3038
3039         /* set clear context state */
3040         radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3041         radeon_ring_write(ring, 0);
3042
3043         /* SQ_VTX_BASE_VTX_LOC */
3044         radeon_ring_write(ring, 0xc0026f00);
3045         radeon_ring_write(ring, 0x00000000);
3046         radeon_ring_write(ring, 0x00000000);
3047         radeon_ring_write(ring, 0x00000000);
3048
3049         /* Clear consts */
3050         radeon_ring_write(ring, 0xc0036f00);
3051         radeon_ring_write(ring, 0x00000bc4);
3052         radeon_ring_write(ring, 0xffffffff);
3053         radeon_ring_write(ring, 0xffffffff);
3054         radeon_ring_write(ring, 0xffffffff);
3055
3056         radeon_ring_write(ring, 0xc0026900);
3057         radeon_ring_write(ring, 0x00000316);
3058         radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3059         radeon_ring_write(ring, 0x00000010); /*  */
3060
3061         radeon_ring_unlock_commit(rdev, ring, false);
3062
3063         return 0;
3064 }
3065
3066 static int evergreen_cp_resume(struct radeon_device *rdev)
3067 {
3068         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3069         u32 tmp;
3070         u32 rb_bufsz;
3071         int r;
3072
3073         /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
3074         WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
3075                                  SOFT_RESET_PA |
3076                                  SOFT_RESET_SH |
3077                                  SOFT_RESET_VGT |
3078                                  SOFT_RESET_SPI |
3079                                  SOFT_RESET_SX));
3080         RREG32(GRBM_SOFT_RESET);
3081         mdelay(15);
3082         WREG32(GRBM_SOFT_RESET, 0);
3083         RREG32(GRBM_SOFT_RESET);
3084
3085         /* Set ring buffer size */
3086         rb_bufsz = order_base_2(ring->ring_size / 8);
3087         tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3088 #ifdef __BIG_ENDIAN
3089         tmp |= BUF_SWAP_32BIT;
3090 #endif
3091         WREG32(CP_RB_CNTL, tmp);
3092         WREG32(CP_SEM_WAIT_TIMER, 0x0);
3093         WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3094
3095         /* Set the write pointer delay */
3096         WREG32(CP_RB_WPTR_DELAY, 0);
3097
3098         /* Initialize the ring buffer's read and write pointers */
3099         WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
3100         WREG32(CP_RB_RPTR_WR, 0);
3101         ring->wptr = 0;
3102         WREG32(CP_RB_WPTR, ring->wptr);
3103
3104         /* set the wb address whether it's enabled or not */
3105         WREG32(CP_RB_RPTR_ADDR,
3106                ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
3107         WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3108         WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3109
3110         if (rdev->wb.enabled)
3111                 WREG32(SCRATCH_UMSK, 0xff);
3112         else {
3113                 tmp |= RB_NO_UPDATE;
3114                 WREG32(SCRATCH_UMSK, 0);
3115         }
3116
3117         mdelay(1);
3118         WREG32(CP_RB_CNTL, tmp);
3119
3120         WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
3121         WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
3122
3123         evergreen_cp_start(rdev);
3124         ring->ready = true;
3125         r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
3126         if (r) {
3127                 ring->ready = false;
3128                 return r;
3129         }
3130         return 0;
3131 }
3132
3133 /*
3134  * Core functions
3135  */
3136 static void evergreen_gpu_init(struct radeon_device *rdev)
3137 {
3138         u32 gb_addr_config;
3139         u32 mc_arb_ramcfg;
3140         u32 sx_debug_1;
3141         u32 smx_dc_ctl0;
3142         u32 sq_config;
3143         u32 sq_lds_resource_mgmt;
3144         u32 sq_gpr_resource_mgmt_1;
3145         u32 sq_gpr_resource_mgmt_2;
3146         u32 sq_gpr_resource_mgmt_3;
3147         u32 sq_thread_resource_mgmt;
3148         u32 sq_thread_resource_mgmt_2;
3149         u32 sq_stack_resource_mgmt_1;
3150         u32 sq_stack_resource_mgmt_2;
3151         u32 sq_stack_resource_mgmt_3;
3152         u32 vgt_cache_invalidation;
3153         u32 hdp_host_path_cntl, tmp;
3154         u32 disabled_rb_mask;
3155         int i, j, ps_thread_count;
3156
3157         switch (rdev->family) {
3158         case CHIP_CYPRESS:
3159         case CHIP_HEMLOCK:
3160                 rdev->config.evergreen.num_ses = 2;
3161                 rdev->config.evergreen.max_pipes = 4;
3162                 rdev->config.evergreen.max_tile_pipes = 8;
3163                 rdev->config.evergreen.max_simds = 10;
3164                 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3165                 rdev->config.evergreen.max_gprs = 256;
3166                 rdev->config.evergreen.max_threads = 248;
3167                 rdev->config.evergreen.max_gs_threads = 32;
3168                 rdev->config.evergreen.max_stack_entries = 512;
3169                 rdev->config.evergreen.sx_num_of_sets = 4;
3170                 rdev->config.evergreen.sx_max_export_size = 256;
3171                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3172                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3173                 rdev->config.evergreen.max_hw_contexts = 8;
3174                 rdev->config.evergreen.sq_num_cf_insts = 2;
3175
3176                 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3177                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3178                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3179                 gb_addr_config = CYPRESS_GB_ADDR_CONFIG_GOLDEN;
3180                 break;
3181         case CHIP_JUNIPER:
3182                 rdev->config.evergreen.num_ses = 1;
3183                 rdev->config.evergreen.max_pipes = 4;
3184                 rdev->config.evergreen.max_tile_pipes = 4;
3185                 rdev->config.evergreen.max_simds = 10;
3186                 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3187                 rdev->config.evergreen.max_gprs = 256;
3188                 rdev->config.evergreen.max_threads = 248;
3189                 rdev->config.evergreen.max_gs_threads = 32;
3190                 rdev->config.evergreen.max_stack_entries = 512;
3191                 rdev->config.evergreen.sx_num_of_sets = 4;
3192                 rdev->config.evergreen.sx_max_export_size = 256;
3193                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3194                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3195                 rdev->config.evergreen.max_hw_contexts = 8;
3196                 rdev->config.evergreen.sq_num_cf_insts = 2;
3197
3198                 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3199                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3200                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3201                 gb_addr_config = JUNIPER_GB_ADDR_CONFIG_GOLDEN;
3202                 break;
3203         case CHIP_REDWOOD:
3204                 rdev->config.evergreen.num_ses = 1;
3205                 rdev->config.evergreen.max_pipes = 4;
3206                 rdev->config.evergreen.max_tile_pipes = 4;
3207                 rdev->config.evergreen.max_simds = 5;
3208                 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3209                 rdev->config.evergreen.max_gprs = 256;
3210                 rdev->config.evergreen.max_threads = 248;
3211                 rdev->config.evergreen.max_gs_threads = 32;
3212                 rdev->config.evergreen.max_stack_entries = 256;
3213                 rdev->config.evergreen.sx_num_of_sets = 4;
3214                 rdev->config.evergreen.sx_max_export_size = 256;
3215                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3216                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3217                 rdev->config.evergreen.max_hw_contexts = 8;
3218                 rdev->config.evergreen.sq_num_cf_insts = 2;
3219
3220                 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3221                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3222                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3223                 gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN;
3224                 break;
3225         case CHIP_CEDAR:
3226         default:
3227                 rdev->config.evergreen.num_ses = 1;
3228                 rdev->config.evergreen.max_pipes = 2;
3229                 rdev->config.evergreen.max_tile_pipes = 2;
3230                 rdev->config.evergreen.max_simds = 2;
3231                 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3232                 rdev->config.evergreen.max_gprs = 256;
3233                 rdev->config.evergreen.max_threads = 192;
3234                 rdev->config.evergreen.max_gs_threads = 16;
3235                 rdev->config.evergreen.max_stack_entries = 256;
3236                 rdev->config.evergreen.sx_num_of_sets = 4;
3237                 rdev->config.evergreen.sx_max_export_size = 128;
3238                 rdev->config.evergreen.sx_max_export_pos_size = 32;
3239                 rdev->config.evergreen.sx_max_export_smx_size = 96;
3240                 rdev->config.evergreen.max_hw_contexts = 4;
3241                 rdev->config.evergreen.sq_num_cf_insts = 1;
3242
3243                 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3244                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3245                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3246                 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3247                 break;
3248         case CHIP_PALM:
3249                 rdev->config.evergreen.num_ses = 1;
3250                 rdev->config.evergreen.max_pipes = 2;
3251                 rdev->config.evergreen.max_tile_pipes = 2;
3252                 rdev->config.evergreen.max_simds = 2;
3253                 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3254                 rdev->config.evergreen.max_gprs = 256;
3255                 rdev->config.evergreen.max_threads = 192;
3256                 rdev->config.evergreen.max_gs_threads = 16;
3257                 rdev->config.evergreen.max_stack_entries = 256;
3258                 rdev->config.evergreen.sx_num_of_sets = 4;
3259                 rdev->config.evergreen.sx_max_export_size = 128;
3260                 rdev->config.evergreen.sx_max_export_pos_size = 32;
3261                 rdev->config.evergreen.sx_max_export_smx_size = 96;
3262                 rdev->config.evergreen.max_hw_contexts = 4;
3263                 rdev->config.evergreen.sq_num_cf_insts = 1;
3264
3265                 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3266                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3267                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3268                 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3269                 break;
3270         case CHIP_SUMO:
3271                 rdev->config.evergreen.num_ses = 1;
3272                 rdev->config.evergreen.max_pipes = 4;
3273                 rdev->config.evergreen.max_tile_pipes = 4;
3274                 if (rdev->pdev->device == 0x9648)
3275                         rdev->config.evergreen.max_simds = 3;
3276                 else if ((rdev->pdev->device == 0x9647) ||
3277                          (rdev->pdev->device == 0x964a))
3278                         rdev->config.evergreen.max_simds = 4;
3279                 else
3280                         rdev->config.evergreen.max_simds = 5;
3281                 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3282                 rdev->config.evergreen.max_gprs = 256;
3283                 rdev->config.evergreen.max_threads = 248;
3284                 rdev->config.evergreen.max_gs_threads = 32;
3285                 rdev->config.evergreen.max_stack_entries = 256;
3286                 rdev->config.evergreen.sx_num_of_sets = 4;
3287                 rdev->config.evergreen.sx_max_export_size = 256;
3288                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3289                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3290                 rdev->config.evergreen.max_hw_contexts = 8;
3291                 rdev->config.evergreen.sq_num_cf_insts = 2;
3292
3293                 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3294                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3295                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3296                 gb_addr_config = SUMO_GB_ADDR_CONFIG_GOLDEN;
3297                 break;
3298         case CHIP_SUMO2:
3299                 rdev->config.evergreen.num_ses = 1;
3300                 rdev->config.evergreen.max_pipes = 4;
3301                 rdev->config.evergreen.max_tile_pipes = 4;
3302                 rdev->config.evergreen.max_simds = 2;
3303                 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3304                 rdev->config.evergreen.max_gprs = 256;
3305                 rdev->config.evergreen.max_threads = 248;
3306                 rdev->config.evergreen.max_gs_threads = 32;
3307                 rdev->config.evergreen.max_stack_entries = 512;
3308                 rdev->config.evergreen.sx_num_of_sets = 4;
3309                 rdev->config.evergreen.sx_max_export_size = 256;
3310                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3311                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3312                 rdev->config.evergreen.max_hw_contexts = 4;
3313                 rdev->config.evergreen.sq_num_cf_insts = 2;
3314
3315                 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3316                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3317                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3318                 gb_addr_config = SUMO2_GB_ADDR_CONFIG_GOLDEN;
3319                 break;
3320         case CHIP_BARTS:
3321                 rdev->config.evergreen.num_ses = 2;
3322                 rdev->config.evergreen.max_pipes = 4;
3323                 rdev->config.evergreen.max_tile_pipes = 8;
3324                 rdev->config.evergreen.max_simds = 7;
3325                 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3326                 rdev->config.evergreen.max_gprs = 256;
3327                 rdev->config.evergreen.max_threads = 248;
3328                 rdev->config.evergreen.max_gs_threads = 32;
3329                 rdev->config.evergreen.max_stack_entries = 512;
3330                 rdev->config.evergreen.sx_num_of_sets = 4;
3331                 rdev->config.evergreen.sx_max_export_size = 256;
3332                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3333                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3334                 rdev->config.evergreen.max_hw_contexts = 8;
3335                 rdev->config.evergreen.sq_num_cf_insts = 2;
3336
3337                 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3338                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3339                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3340                 gb_addr_config = BARTS_GB_ADDR_CONFIG_GOLDEN;
3341                 break;
3342         case CHIP_TURKS:
3343                 rdev->config.evergreen.num_ses = 1;
3344                 rdev->config.evergreen.max_pipes = 4;
3345                 rdev->config.evergreen.max_tile_pipes = 4;
3346                 rdev->config.evergreen.max_simds = 6;
3347                 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3348                 rdev->config.evergreen.max_gprs = 256;
3349                 rdev->config.evergreen.max_threads = 248;
3350                 rdev->config.evergreen.max_gs_threads = 32;
3351                 rdev->config.evergreen.max_stack_entries = 256;
3352                 rdev->config.evergreen.sx_num_of_sets = 4;
3353                 rdev->config.evergreen.sx_max_export_size = 256;
3354                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3355                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3356                 rdev->config.evergreen.max_hw_contexts = 8;
3357                 rdev->config.evergreen.sq_num_cf_insts = 2;
3358
3359                 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3360                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3361                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3362                 gb_addr_config = TURKS_GB_ADDR_CONFIG_GOLDEN;
3363                 break;
3364         case CHIP_CAICOS:
3365                 rdev->config.evergreen.num_ses = 1;
3366                 rdev->config.evergreen.max_pipes = 2;
3367                 rdev->config.evergreen.max_tile_pipes = 2;
3368                 rdev->config.evergreen.max_simds = 2;
3369                 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3370                 rdev->config.evergreen.max_gprs = 256;
3371                 rdev->config.evergreen.max_threads = 192;
3372                 rdev->config.evergreen.max_gs_threads = 16;
3373                 rdev->config.evergreen.max_stack_entries = 256;
3374                 rdev->config.evergreen.sx_num_of_sets = 4;
3375                 rdev->config.evergreen.sx_max_export_size = 128;
3376                 rdev->config.evergreen.sx_max_export_pos_size = 32;
3377                 rdev->config.evergreen.sx_max_export_smx_size = 96;
3378                 rdev->config.evergreen.max_hw_contexts = 4;
3379                 rdev->config.evergreen.sq_num_cf_insts = 1;
3380
3381                 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3382                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3383                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3384                 gb_addr_config = CAICOS_GB_ADDR_CONFIG_GOLDEN;
3385                 break;
3386         }
3387
3388         /* Initialize HDP */
3389         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3390                 WREG32((0x2c14 + j), 0x00000000);
3391                 WREG32((0x2c18 + j), 0x00000000);
3392                 WREG32((0x2c1c + j), 0x00000000);
3393                 WREG32((0x2c20 + j), 0x00000000);
3394                 WREG32((0x2c24 + j), 0x00000000);
3395         }
3396
3397         WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3398         WREG32(SRBM_INT_CNTL, 0x1);
3399         WREG32(SRBM_INT_ACK, 0x1);
3400
3401         evergreen_fix_pci_max_read_req_size(rdev);
3402
3403         RREG32(MC_SHARED_CHMAP);
3404         if ((rdev->family == CHIP_PALM) ||
3405             (rdev->family == CHIP_SUMO) ||
3406             (rdev->family == CHIP_SUMO2))
3407                 mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
3408         else
3409                 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3410
3411         /* setup tiling info dword.  gb_addr_config is not adequate since it does
3412          * not have bank info, so create a custom tiling dword.
3413          * bits 3:0   num_pipes
3414          * bits 7:4   num_banks
3415          * bits 11:8  group_size
3416          * bits 15:12 row_size
3417          */
3418         rdev->config.evergreen.tile_config = 0;
3419         switch (rdev->config.evergreen.max_tile_pipes) {
3420         case 1:
3421         default:
3422                 rdev->config.evergreen.tile_config |= (0 << 0);
3423                 break;
3424         case 2:
3425                 rdev->config.evergreen.tile_config |= (1 << 0);
3426                 break;
3427         case 4:
3428                 rdev->config.evergreen.tile_config |= (2 << 0);
3429                 break;
3430         case 8:
3431                 rdev->config.evergreen.tile_config |= (3 << 0);
3432                 break;
3433         }
3434         /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
3435         if (rdev->flags & RADEON_IS_IGP)
3436                 rdev->config.evergreen.tile_config |= 1 << 4;
3437         else {
3438                 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3439                 case 0: /* four banks */
3440                         rdev->config.evergreen.tile_config |= 0 << 4;
3441                         break;
3442                 case 1: /* eight banks */
3443                         rdev->config.evergreen.tile_config |= 1 << 4;
3444                         break;
3445                 case 2: /* sixteen banks */
3446                 default:
3447                         rdev->config.evergreen.tile_config |= 2 << 4;
3448                         break;
3449                 }
3450         }
3451         rdev->config.evergreen.tile_config |= 0 << 8;
3452         rdev->config.evergreen.tile_config |=
3453                 ((gb_addr_config & 0x30000000) >> 28) << 12;
3454
3455         if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) {
3456                 u32 efuse_straps_4;
3457                 u32 efuse_straps_3;
3458
3459                 efuse_straps_4 = RREG32_RCU(0x204);
3460                 efuse_straps_3 = RREG32_RCU(0x203);
3461                 tmp = (((efuse_straps_4 & 0xf) << 4) |
3462                       ((efuse_straps_3 & 0xf0000000) >> 28));
3463         } else {
3464                 tmp = 0;
3465                 for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) {
3466                         u32 rb_disable_bitmap;
3467
3468                         WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3469                         WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3470                         rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
3471                         tmp <<= 4;
3472                         tmp |= rb_disable_bitmap;
3473                 }
3474         }
3475         /* enabled rb are just the one not disabled :) */
3476         disabled_rb_mask = tmp;
3477         tmp = 0;
3478         for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3479                 tmp |= (1 << i);
3480         /* if all the backends are disabled, fix it up here */
3481         if ((disabled_rb_mask & tmp) == tmp) {
3482                 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3483                         disabled_rb_mask &= ~(1 << i);
3484         }
3485
3486         for (i = 0; i < rdev->config.evergreen.num_ses; i++) {
3487                 u32 simd_disable_bitmap;
3488
3489                 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3490                 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3491                 simd_disable_bitmap = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
3492                 simd_disable_bitmap |= 0xffffffff << rdev->config.evergreen.max_simds;
3493                 tmp <<= 16;
3494                 tmp |= simd_disable_bitmap;
3495         }
3496         rdev->config.evergreen.active_simds = hweight32(~tmp);
3497
3498         WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3499         WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3500
3501         WREG32(GB_ADDR_CONFIG, gb_addr_config);
3502         WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3503         WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3504         WREG32(DMA_TILING_CONFIG, gb_addr_config);
3505         WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3506         WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3507         WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3508
3509         if ((rdev->config.evergreen.max_backends == 1) &&
3510             (rdev->flags & RADEON_IS_IGP)) {
3511                 if ((disabled_rb_mask & 3) == 1) {
3512                         /* RB0 disabled, RB1 enabled */
3513                         tmp = 0x11111111;
3514                 } else {
3515                         /* RB1 disabled, RB0 enabled */
3516                         tmp = 0x00000000;
3517                 }
3518         } else {
3519                 tmp = gb_addr_config & NUM_PIPES_MASK;
3520                 tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends,
3521                                                 EVERGREEN_MAX_BACKENDS, disabled_rb_mask);
3522         }
3523         rdev->config.evergreen.backend_map = tmp;
3524         WREG32(GB_BACKEND_MAP, tmp);
3525
3526         WREG32(CGTS_SYS_TCC_DISABLE, 0);
3527         WREG32(CGTS_TCC_DISABLE, 0);
3528         WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
3529         WREG32(CGTS_USER_TCC_DISABLE, 0);
3530
3531         /* set HW defaults for 3D engine */
3532         WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3533                                      ROQ_IB2_START(0x2b)));
3534
3535         WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
3536
3537         WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO |
3538                              SYNC_GRADIENT |
3539                              SYNC_WALKER |
3540                              SYNC_ALIGNER));
3541
3542         sx_debug_1 = RREG32(SX_DEBUG_1);
3543         sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
3544         WREG32(SX_DEBUG_1, sx_debug_1);
3545
3546
3547         smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
3548         smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
3549         smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
3550         WREG32(SMX_DC_CTL0, smx_dc_ctl0);
3551
3552         if (rdev->family <= CHIP_SUMO2)
3553                 WREG32(SMX_SAR_CTL0, 0x00010000);
3554
3555         WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) |
3556                                         POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) |
3557                                         SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
3558
3559         WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) |
3560                                  SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) |
3561                                  SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
3562
3563         WREG32(VGT_NUM_INSTANCES, 1);
3564         WREG32(SPI_CONFIG_CNTL, 0);
3565         WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3566         WREG32(CP_PERFMON_CNTL, 0);
3567
3568         WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) |
3569                                   FETCH_FIFO_HIWATER(0x4) |
3570                                   DONE_FIFO_HIWATER(0xe0) |
3571                                   ALU_UPDATE_FIFO_HIWATER(0x8)));
3572
3573         sq_config = RREG32(SQ_CONFIG);
3574         sq_config &= ~(PS_PRIO(3) |
3575                        VS_PRIO(3) |
3576                        GS_PRIO(3) |
3577                        ES_PRIO(3));
3578         sq_config |= (VC_ENABLE |
3579                       EXPORT_SRC_C |
3580                       PS_PRIO(0) |
3581                       VS_PRIO(1) |
3582                       GS_PRIO(2) |
3583                       ES_PRIO(3));
3584
3585         switch (rdev->family) {
3586         case CHIP_CEDAR:
3587         case CHIP_PALM:
3588         case CHIP_SUMO:
3589         case CHIP_SUMO2:
3590         case CHIP_CAICOS:
3591                 /* no vertex cache */
3592                 sq_config &= ~VC_ENABLE;
3593                 break;
3594         default:
3595                 break;
3596         }
3597
3598         sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
3599
3600         sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32);
3601         sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
3602         sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4);
3603         sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3604         sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3605         sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3606         sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3607
3608         switch (rdev->family) {
3609         case CHIP_CEDAR:
3610         case CHIP_PALM:
3611         case CHIP_SUMO:
3612         case CHIP_SUMO2:
3613                 ps_thread_count = 96;
3614                 break;
3615         default:
3616                 ps_thread_count = 128;
3617                 break;
3618         }
3619
3620         sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
3621         sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3622         sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3623         sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3624         sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3625         sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3626
3627         sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3628         sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3629         sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3630         sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3631         sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3632         sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3633
3634         WREG32(SQ_CONFIG, sq_config);
3635         WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
3636         WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
3637         WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
3638         WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
3639         WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
3640         WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
3641         WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
3642         WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
3643         WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
3644         WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
3645
3646         WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3647                                           FORCE_EOV_MAX_REZ_CNT(255)));
3648
3649         switch (rdev->family) {
3650         case CHIP_CEDAR:
3651         case CHIP_PALM:
3652         case CHIP_SUMO:
3653         case CHIP_SUMO2:
3654         case CHIP_CAICOS:
3655                 vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
3656                 break;
3657         default:
3658                 vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
3659                 break;
3660         }
3661         vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO);
3662         WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
3663
3664         WREG32(VGT_GS_VERTEX_REUSE, 16);
3665         WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
3666         WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3667
3668         WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
3669         WREG32(VGT_OUT_DEALLOC_CNTL, 16);
3670
3671         WREG32(CB_PERF_CTR0_SEL_0, 0);
3672         WREG32(CB_PERF_CTR0_SEL_1, 0);
3673         WREG32(CB_PERF_CTR1_SEL_0, 0);
3674         WREG32(CB_PERF_CTR1_SEL_1, 0);
3675         WREG32(CB_PERF_CTR2_SEL_0, 0);
3676         WREG32(CB_PERF_CTR2_SEL_1, 0);
3677         WREG32(CB_PERF_CTR3_SEL_0, 0);
3678         WREG32(CB_PERF_CTR3_SEL_1, 0);
3679
3680         /* clear render buffer base addresses */
3681         WREG32(CB_COLOR0_BASE, 0);
3682         WREG32(CB_COLOR1_BASE, 0);
3683         WREG32(CB_COLOR2_BASE, 0);
3684         WREG32(CB_COLOR3_BASE, 0);
3685         WREG32(CB_COLOR4_BASE, 0);
3686         WREG32(CB_COLOR5_BASE, 0);
3687         WREG32(CB_COLOR6_BASE, 0);
3688         WREG32(CB_COLOR7_BASE, 0);
3689         WREG32(CB_COLOR8_BASE, 0);
3690         WREG32(CB_COLOR9_BASE, 0);
3691         WREG32(CB_COLOR10_BASE, 0);
3692         WREG32(CB_COLOR11_BASE, 0);
3693
3694         /* set the shader const cache sizes to 0 */
3695         for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
3696                 WREG32(i, 0);
3697         for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
3698                 WREG32(i, 0);
3699
3700         tmp = RREG32(HDP_MISC_CNTL);
3701         tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3702         WREG32(HDP_MISC_CNTL, tmp);
3703
3704         hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3705         WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3706
3707         WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3708
3709         udelay(50);
3710
3711 }
3712
3713 int evergreen_mc_init(struct radeon_device *rdev)
3714 {
3715         u32 tmp;
3716         int chansize, numchan;
3717
3718         /* Get VRAM informations */
3719         rdev->mc.vram_is_ddr = true;
3720         if ((rdev->family == CHIP_PALM) ||
3721             (rdev->family == CHIP_SUMO) ||
3722             (rdev->family == CHIP_SUMO2))
3723                 tmp = RREG32(FUS_MC_ARB_RAMCFG);
3724         else
3725                 tmp = RREG32(MC_ARB_RAMCFG);
3726         if (tmp & CHANSIZE_OVERRIDE) {
3727                 chansize = 16;
3728         } else if (tmp & CHANSIZE_MASK) {
3729                 chansize = 64;
3730         } else {
3731                 chansize = 32;
3732         }
3733         tmp = RREG32(MC_SHARED_CHMAP);
3734         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3735         case 0:
3736         default:
3737                 numchan = 1;
3738                 break;
3739         case 1:
3740                 numchan = 2;
3741                 break;
3742         case 2:
3743                 numchan = 4;
3744                 break;
3745         case 3:
3746                 numchan = 8;
3747                 break;
3748         }
3749         rdev->mc.vram_width = numchan * chansize;
3750         /* Could aper size report 0 ? */
3751         rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3752         rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
3753         /* Setup GPU memory space */
3754         if ((rdev->family == CHIP_PALM) ||
3755             (rdev->family == CHIP_SUMO) ||
3756             (rdev->family == CHIP_SUMO2)) {
3757                 /* size in bytes on fusion */
3758                 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
3759                 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
3760         } else {
3761                 /* size in MB on evergreen/cayman/tn */
3762                 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3763                 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3764         }
3765         rdev->mc.visible_vram_size = rdev->mc.aper_size;
3766         r700_vram_gtt_location(rdev, &rdev->mc);
3767         radeon_update_bandwidth_info(rdev);
3768
3769         return 0;
3770 }
3771
3772 void evergreen_print_gpu_status_regs(struct radeon_device *rdev)
3773 {
3774         dev_info(rdev->dev, "  GRBM_STATUS               = 0x%08X\n",
3775                 RREG32(GRBM_STATUS));
3776         dev_info(rdev->dev, "  GRBM_STATUS_SE0           = 0x%08X\n",
3777                 RREG32(GRBM_STATUS_SE0));
3778         dev_info(rdev->dev, "  GRBM_STATUS_SE1           = 0x%08X\n",
3779                 RREG32(GRBM_STATUS_SE1));
3780         dev_info(rdev->dev, "  SRBM_STATUS               = 0x%08X\n",
3781                 RREG32(SRBM_STATUS));
3782         dev_info(rdev->dev, "  SRBM_STATUS2              = 0x%08X\n",
3783                 RREG32(SRBM_STATUS2));
3784         dev_info(rdev->dev, "  R_008674_CP_STALLED_STAT1 = 0x%08X\n",
3785                 RREG32(CP_STALLED_STAT1));
3786         dev_info(rdev->dev, "  R_008678_CP_STALLED_STAT2 = 0x%08X\n",
3787                 RREG32(CP_STALLED_STAT2));
3788         dev_info(rdev->dev, "  R_00867C_CP_BUSY_STAT     = 0x%08X\n",
3789                 RREG32(CP_BUSY_STAT));
3790         dev_info(rdev->dev, "  R_008680_CP_STAT          = 0x%08X\n",
3791                 RREG32(CP_STAT));
3792         dev_info(rdev->dev, "  R_00D034_DMA_STATUS_REG   = 0x%08X\n",
3793                 RREG32(DMA_STATUS_REG));
3794         if (rdev->family >= CHIP_CAYMAN) {
3795                 dev_info(rdev->dev, "  R_00D834_DMA_STATUS_REG   = 0x%08X\n",
3796                          RREG32(DMA_STATUS_REG + 0x800));
3797         }
3798 }
3799
3800 bool evergreen_is_display_hung(struct radeon_device *rdev)
3801 {
3802         u32 crtc_hung = 0;
3803         u32 crtc_status[6];
3804         u32 i, j, tmp;
3805
3806         for (i = 0; i < rdev->num_crtc; i++) {
3807                 if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN) {
3808                         crtc_status[i] = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3809                         crtc_hung |= (1 << i);
3810                 }
3811         }
3812
3813         for (j = 0; j < 10; j++) {
3814                 for (i = 0; i < rdev->num_crtc; i++) {
3815                         if (crtc_hung & (1 << i)) {
3816                                 tmp = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3817                                 if (tmp != crtc_status[i])
3818                                         crtc_hung &= ~(1 << i);
3819                         }
3820                 }
3821                 if (crtc_hung == 0)
3822                         return false;
3823                 udelay(100);
3824         }
3825
3826         return true;
3827 }
3828
3829 u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev)
3830 {
3831         u32 reset_mask = 0;
3832         u32 tmp;
3833
3834         /* GRBM_STATUS */
3835         tmp = RREG32(GRBM_STATUS);
3836         if (tmp & (PA_BUSY | SC_BUSY |
3837                    SH_BUSY | SX_BUSY |
3838                    TA_BUSY | VGT_BUSY |
3839                    DB_BUSY | CB_BUSY |
3840                    SPI_BUSY | VGT_BUSY_NO_DMA))
3841                 reset_mask |= RADEON_RESET_GFX;
3842
3843         if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3844                    CP_BUSY | CP_COHERENCY_BUSY))
3845                 reset_mask |= RADEON_RESET_CP;
3846
3847         if (tmp & GRBM_EE_BUSY)
3848                 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3849
3850         /* DMA_STATUS_REG */
3851         tmp = RREG32(DMA_STATUS_REG);
3852         if (!(tmp & DMA_IDLE))
3853                 reset_mask |= RADEON_RESET_DMA;
3854
3855         /* SRBM_STATUS2 */
3856         tmp = RREG32(SRBM_STATUS2);
3857         if (tmp & DMA_BUSY)
3858                 reset_mask |= RADEON_RESET_DMA;
3859
3860         /* SRBM_STATUS */
3861         tmp = RREG32(SRBM_STATUS);
3862         if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3863                 reset_mask |= RADEON_RESET_RLC;
3864
3865         if (tmp & IH_BUSY)
3866                 reset_mask |= RADEON_RESET_IH;
3867
3868         if (tmp & SEM_BUSY)
3869                 reset_mask |= RADEON_RESET_SEM;
3870
3871         if (tmp & GRBM_RQ_PENDING)
3872                 reset_mask |= RADEON_RESET_GRBM;
3873
3874         if (tmp & VMC_BUSY)
3875                 reset_mask |= RADEON_RESET_VMC;
3876
3877         if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3878                    MCC_BUSY | MCD_BUSY))
3879                 reset_mask |= RADEON_RESET_MC;
3880
3881         if (evergreen_is_display_hung(rdev))
3882                 reset_mask |= RADEON_RESET_DISPLAY;
3883
3884         /* VM_L2_STATUS */
3885         tmp = RREG32(VM_L2_STATUS);
3886         if (tmp & L2_BUSY)
3887                 reset_mask |= RADEON_RESET_VMC;
3888
3889         /* Skip MC reset as it's mostly likely not hung, just busy */
3890         if (reset_mask & RADEON_RESET_MC) {
3891                 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3892                 reset_mask &= ~RADEON_RESET_MC;
3893         }
3894
3895         return reset_mask;
3896 }
3897
3898 static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3899 {
3900         struct evergreen_mc_save save;
3901         u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3902         u32 tmp;
3903
3904         if (reset_mask == 0)
3905                 return;
3906
3907         dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3908
3909         evergreen_print_gpu_status_regs(rdev);
3910
3911         /* Disable CP parsing/prefetching */
3912         WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3913
3914         if (reset_mask & RADEON_RESET_DMA) {
3915                 /* Disable DMA */
3916                 tmp = RREG32(DMA_RB_CNTL);
3917                 tmp &= ~DMA_RB_ENABLE;
3918                 WREG32(DMA_RB_CNTL, tmp);
3919         }
3920
3921         udelay(50);
3922
3923         evergreen_mc_stop(rdev, &save);
3924         if (evergreen_mc_wait_for_idle(rdev)) {
3925                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3926         }
3927
3928         if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
3929                 grbm_soft_reset |= SOFT_RESET_DB |
3930                         SOFT_RESET_CB |
3931                         SOFT_RESET_PA |
3932                         SOFT_RESET_SC |
3933                         SOFT_RESET_SPI |
3934                         SOFT_RESET_SX |
3935                         SOFT_RESET_SH |
3936                         SOFT_RESET_TC |
3937                         SOFT_RESET_TA |
3938                         SOFT_RESET_VC |
3939                         SOFT_RESET_VGT;
3940         }
3941
3942         if (reset_mask & RADEON_RESET_CP) {
3943                 grbm_soft_reset |= SOFT_RESET_CP |
3944                         SOFT_RESET_VGT;
3945
3946                 srbm_soft_reset |= SOFT_RESET_GRBM;
3947         }
3948
3949         if (reset_mask & RADEON_RESET_DMA)
3950                 srbm_soft_reset |= SOFT_RESET_DMA;
3951
3952         if (reset_mask & RADEON_RESET_DISPLAY)
3953                 srbm_soft_reset |= SOFT_RESET_DC;
3954
3955         if (reset_mask & RADEON_RESET_RLC)
3956                 srbm_soft_reset |= SOFT_RESET_RLC;
3957
3958         if (reset_mask & RADEON_RESET_SEM)
3959                 srbm_soft_reset |= SOFT_RESET_SEM;
3960
3961         if (reset_mask & RADEON_RESET_IH)
3962                 srbm_soft_reset |= SOFT_RESET_IH;
3963
3964         if (reset_mask & RADEON_RESET_GRBM)
3965                 srbm_soft_reset |= SOFT_RESET_GRBM;
3966
3967         if (reset_mask & RADEON_RESET_VMC)
3968                 srbm_soft_reset |= SOFT_RESET_VMC;
3969
3970         if (!(rdev->flags & RADEON_IS_IGP)) {
3971                 if (reset_mask & RADEON_RESET_MC)
3972                         srbm_soft_reset |= SOFT_RESET_MC;
3973         }
3974
3975         if (grbm_soft_reset) {
3976                 tmp = RREG32(GRBM_SOFT_RESET);
3977                 tmp |= grbm_soft_reset;
3978                 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3979                 WREG32(GRBM_SOFT_RESET, tmp);
3980                 tmp = RREG32(GRBM_SOFT_RESET);
3981
3982                 udelay(50);
3983
3984                 tmp &= ~grbm_soft_reset;
3985                 WREG32(GRBM_SOFT_RESET, tmp);
3986                 tmp = RREG32(GRBM_SOFT_RESET);
3987         }
3988
3989         if (srbm_soft_reset) {
3990                 tmp = RREG32(SRBM_SOFT_RESET);
3991                 tmp |= srbm_soft_reset;
3992                 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3993                 WREG32(SRBM_SOFT_RESET, tmp);
3994                 tmp = RREG32(SRBM_SOFT_RESET);
3995
3996                 udelay(50);
3997
3998                 tmp &= ~srbm_soft_reset;
3999                 WREG32(SRBM_SOFT_RESET, tmp);
4000                 tmp = RREG32(SRBM_SOFT_RESET);
4001         }
4002
4003         /* Wait a little for things to settle down */
4004         udelay(50);
4005
4006         evergreen_mc_resume(rdev, &save);
4007         udelay(50);
4008
4009         evergreen_print_gpu_status_regs(rdev);
4010 }
4011
4012 void evergreen_gpu_pci_config_reset(struct radeon_device *rdev)
4013 {
4014         struct evergreen_mc_save save;
4015         u32 tmp, i;
4016
4017         dev_info(rdev->dev, "GPU pci config reset\n");
4018
4019         /* disable dpm? */
4020
4021         /* Disable CP parsing/prefetching */
4022         WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
4023         udelay(50);
4024         /* Disable DMA */
4025         tmp = RREG32(DMA_RB_CNTL);
4026         tmp &= ~DMA_RB_ENABLE;
4027         WREG32(DMA_RB_CNTL, tmp);
4028         /* XXX other engines? */
4029
4030         /* halt the rlc */
4031         r600_rlc_stop(rdev);
4032
4033         udelay(50);
4034
4035         /* set mclk/sclk to bypass */
4036         rv770_set_clk_bypass_mode(rdev);
4037         /* disable BM */
4038         pci_clear_master(rdev->pdev);
4039         /* disable mem access */
4040         evergreen_mc_stop(rdev, &save);
4041         if (evergreen_mc_wait_for_idle(rdev)) {
4042                 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
4043         }
4044         /* reset */
4045         radeon_pci_config_reset(rdev);
4046         /* wait for asic to come out of reset */
4047         for (i = 0; i < rdev->usec_timeout; i++) {
4048                 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
4049                         break;
4050                 udelay(1);
4051         }
4052 }
4053
4054 int evergreen_asic_reset(struct radeon_device *rdev, bool hard)
4055 {
4056         u32 reset_mask;
4057
4058         if (hard) {
4059                 evergreen_gpu_pci_config_reset(rdev);
4060                 return 0;
4061         }
4062
4063         reset_mask = evergreen_gpu_check_soft_reset(rdev);
4064
4065         if (reset_mask)
4066                 r600_set_bios_scratch_engine_hung(rdev, true);
4067
4068         /* try soft reset */
4069         evergreen_gpu_soft_reset(rdev, reset_mask);
4070
4071         reset_mask = evergreen_gpu_check_soft_reset(rdev);
4072
4073         /* try pci config reset */
4074         if (reset_mask && radeon_hard_reset)
4075                 evergreen_gpu_pci_config_reset(rdev);
4076
4077         reset_mask = evergreen_gpu_check_soft_reset(rdev);
4078
4079         if (!reset_mask)
4080                 r600_set_bios_scratch_engine_hung(rdev, false);
4081
4082         return 0;
4083 }
4084
4085 /**
4086  * evergreen_gfx_is_lockup - Check if the GFX engine is locked up
4087  *
4088  * @rdev: radeon_device pointer
4089  * @ring: radeon_ring structure holding ring information
4090  *
4091  * Check if the GFX engine is locked up.
4092  * Returns true if the engine appears to be locked up, false if not.
4093  */
4094 bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
4095 {
4096         u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
4097
4098         if (!(reset_mask & (RADEON_RESET_GFX |
4099                             RADEON_RESET_COMPUTE |
4100                             RADEON_RESET_CP))) {
4101                 radeon_ring_lockup_update(rdev, ring);
4102                 return false;
4103         }
4104         return radeon_ring_test_lockup(rdev, ring);
4105 }
4106
4107 /*
4108  * RLC
4109  */
4110 #define RLC_SAVE_RESTORE_LIST_END_MARKER    0x00000000
4111 #define RLC_CLEAR_STATE_END_MARKER          0x00000001
4112
4113 void sumo_rlc_fini(struct radeon_device *rdev)
4114 {
4115         int r;
4116
4117         /* save restore block */
4118         if (rdev->rlc.save_restore_obj) {
4119                 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4120                 if (unlikely(r != 0))
4121                         dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);
4122                 radeon_bo_unpin(rdev->rlc.save_restore_obj);
4123                 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4124
4125                 radeon_bo_unref(&rdev->rlc.save_restore_obj);
4126                 rdev->rlc.save_restore_obj = NULL;
4127         }
4128
4129         /* clear state block */
4130         if (rdev->rlc.clear_state_obj) {
4131                 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4132                 if (unlikely(r != 0))
4133                         dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);
4134                 radeon_bo_unpin(rdev->rlc.clear_state_obj);
4135                 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4136
4137                 radeon_bo_unref(&rdev->rlc.clear_state_obj);
4138                 rdev->rlc.clear_state_obj = NULL;
4139         }
4140
4141         /* clear state block */
4142         if (rdev->rlc.cp_table_obj) {
4143                 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4144                 if (unlikely(r != 0))
4145                         dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4146                 radeon_bo_unpin(rdev->rlc.cp_table_obj);
4147                 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4148
4149                 radeon_bo_unref(&rdev->rlc.cp_table_obj);
4150                 rdev->rlc.cp_table_obj = NULL;
4151         }
4152 }
4153
4154 #define CP_ME_TABLE_SIZE    96
4155
4156 int sumo_rlc_init(struct radeon_device *rdev)
4157 {
4158         const u32 *src_ptr;
4159         volatile u32 *dst_ptr;
4160         u32 dws, data, i, j, k, reg_num;
4161         u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index = 0;
4162         u64 reg_list_mc_addr;
4163         const struct cs_section_def *cs_data;
4164         int r;
4165
4166         src_ptr = rdev->rlc.reg_list;
4167         dws = rdev->rlc.reg_list_size;
4168         if (rdev->family >= CHIP_BONAIRE) {
4169                 dws += (5 * 16) + 48 + 48 + 64;
4170         }
4171         cs_data = rdev->rlc.cs_data;
4172
4173         if (src_ptr) {
4174                 /* save restore block */
4175                 if (rdev->rlc.save_restore_obj == NULL) {
4176                         r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
4177                                              RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4178                                              NULL, &rdev->rlc.save_restore_obj);
4179                         if (r) {
4180                                 dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);
4181                                 return r;
4182                         }
4183                 }
4184
4185                 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4186                 if (unlikely(r != 0)) {
4187                         sumo_rlc_fini(rdev);
4188                         return r;
4189                 }
4190                 r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
4191                                   &rdev->rlc.save_restore_gpu_addr);
4192                 if (r) {
4193                         radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4194                         dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
4195                         sumo_rlc_fini(rdev);
4196                         return r;
4197                 }
4198
4199                 r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void **)&rdev->rlc.sr_ptr);
4200                 if (r) {
4201                         dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r);
4202                         sumo_rlc_fini(rdev);
4203                         return r;
4204                 }
4205                 /* write the sr buffer */
4206                 dst_ptr = rdev->rlc.sr_ptr;
4207                 if (rdev->family >= CHIP_TAHITI) {
4208                         /* SI */
4209                         for (i = 0; i < rdev->rlc.reg_list_size; i++)
4210                                 dst_ptr[i] = cpu_to_le32(src_ptr[i]);
4211                 } else {
4212                         /* ON/LN/TN */
4213                         /* format:
4214                          * dw0: (reg2 << 16) | reg1
4215                          * dw1: reg1 save space
4216                          * dw2: reg2 save space
4217                          */
4218                         for (i = 0; i < dws; i++) {
4219                                 data = src_ptr[i] >> 2;
4220                                 i++;
4221                                 if (i < dws)
4222                                         data |= (src_ptr[i] >> 2) << 16;
4223                                 j = (((i - 1) * 3) / 2);
4224                                 dst_ptr[j] = cpu_to_le32(data);
4225                         }
4226                         j = ((i * 3) / 2);
4227                         dst_ptr[j] = cpu_to_le32(RLC_SAVE_RESTORE_LIST_END_MARKER);
4228                 }
4229                 radeon_bo_kunmap(rdev->rlc.save_restore_obj);
4230                 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4231         }
4232
4233         if (cs_data) {
4234                 /* clear state block */
4235                 if (rdev->family >= CHIP_BONAIRE) {
4236                         rdev->rlc.clear_state_size = dws = cik_get_csb_size(rdev);
4237                 } else if (rdev->family >= CHIP_TAHITI) {
4238                         rdev->rlc.clear_state_size = si_get_csb_size(rdev);
4239                         dws = rdev->rlc.clear_state_size + (256 / 4);
4240                 } else {
4241                         reg_list_num = 0;
4242                         dws = 0;
4243                         for (i = 0; cs_data[i].section != NULL; i++) {
4244                                 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4245                                         reg_list_num++;
4246                                         dws += cs_data[i].section[j].reg_count;
4247                                 }
4248                         }
4249                         reg_list_blk_index = (3 * reg_list_num + 2);
4250                         dws += reg_list_blk_index;
4251                         rdev->rlc.clear_state_size = dws;
4252                 }
4253
4254                 if (rdev->rlc.clear_state_obj == NULL) {
4255                         r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
4256                                              RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4257                                              NULL, &rdev->rlc.clear_state_obj);
4258                         if (r) {
4259                                 dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
4260                                 sumo_rlc_fini(rdev);
4261                                 return r;
4262                         }
4263                 }
4264                 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4265                 if (unlikely(r != 0)) {
4266                         sumo_rlc_fini(rdev);
4267                         return r;
4268                 }
4269                 r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
4270                                   &rdev->rlc.clear_state_gpu_addr);
4271                 if (r) {
4272                         radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4273                         dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
4274                         sumo_rlc_fini(rdev);
4275                         return r;
4276                 }
4277
4278                 r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void **)&rdev->rlc.cs_ptr);
4279                 if (r) {
4280                         dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r);
4281                         sumo_rlc_fini(rdev);
4282                         return r;
4283                 }
4284                 /* set up the cs buffer */
4285                 dst_ptr = rdev->rlc.cs_ptr;
4286                 if (rdev->family >= CHIP_BONAIRE) {
4287                         cik_get_csb_buffer(rdev, dst_ptr);
4288                 } else if (rdev->family >= CHIP_TAHITI) {
4289                         reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + 256;
4290                         dst_ptr[0] = cpu_to_le32(upper_32_bits(reg_list_mc_addr));
4291                         dst_ptr[1] = cpu_to_le32(lower_32_bits(reg_list_mc_addr));
4292                         dst_ptr[2] = cpu_to_le32(rdev->rlc.clear_state_size);
4293                         si_get_csb_buffer(rdev, &dst_ptr[(256/4)]);
4294                 } else {
4295                         reg_list_hdr_blk_index = 0;
4296                         reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4);
4297                         data = upper_32_bits(reg_list_mc_addr);
4298                         dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4299                         reg_list_hdr_blk_index++;
4300                         for (i = 0; cs_data[i].section != NULL; i++) {
4301                                 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4302                                         reg_num = cs_data[i].section[j].reg_count;
4303                                         data = reg_list_mc_addr & 0xffffffff;
4304                                         dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4305                                         reg_list_hdr_blk_index++;
4306
4307                                         data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff;
4308                                         dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4309                                         reg_list_hdr_blk_index++;
4310
4311                                         data = 0x08000000 | (reg_num * 4);
4312                                         dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4313                                         reg_list_hdr_blk_index++;
4314
4315                                         for (k = 0; k < reg_num; k++) {
4316                                                 data = cs_data[i].section[j].extent[k];
4317                                                 dst_ptr[reg_list_blk_index + k] = cpu_to_le32(data);
4318                                         }
4319                                         reg_list_mc_addr += reg_num * 4;
4320                                         reg_list_blk_index += reg_num;
4321                                 }
4322                         }
4323                         dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(RLC_CLEAR_STATE_END_MARKER);
4324                 }
4325                 radeon_bo_kunmap(rdev->rlc.clear_state_obj);
4326                 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4327         }
4328
4329         if (rdev->rlc.cp_table_size) {
4330                 if (rdev->rlc.cp_table_obj == NULL) {
4331                         r = radeon_bo_create(rdev, rdev->rlc.cp_table_size,
4332                                              PAGE_SIZE, true,
4333                                              RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4334                                              NULL, &rdev->rlc.cp_table_obj);
4335                         if (r) {
4336                                 dev_warn(rdev->dev, "(%d) create RLC cp table bo failed\n", r);
4337                                 sumo_rlc_fini(rdev);
4338                                 return r;
4339                         }
4340                 }
4341
4342                 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4343                 if (unlikely(r != 0)) {
4344                         dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4345                         sumo_rlc_fini(rdev);
4346                         return r;
4347                 }
4348                 r = radeon_bo_pin(rdev->rlc.cp_table_obj, RADEON_GEM_DOMAIN_VRAM,
4349                                   &rdev->rlc.cp_table_gpu_addr);
4350                 if (r) {
4351                         radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4352                         dev_warn(rdev->dev, "(%d) pin RLC cp_table bo failed\n", r);
4353                         sumo_rlc_fini(rdev);
4354                         return r;
4355                 }
4356                 r = radeon_bo_kmap(rdev->rlc.cp_table_obj, (void **)&rdev->rlc.cp_table_ptr);
4357                 if (r) {
4358                         dev_warn(rdev->dev, "(%d) map RLC cp table bo failed\n", r);
4359                         sumo_rlc_fini(rdev);
4360                         return r;
4361                 }
4362
4363                 cik_init_cp_pg_table(rdev);
4364
4365                 radeon_bo_kunmap(rdev->rlc.cp_table_obj);
4366                 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4367
4368         }
4369
4370         return 0;
4371 }
4372
4373 static void evergreen_rlc_start(struct radeon_device *rdev)
4374 {
4375         u32 mask = RLC_ENABLE;
4376
4377         if (rdev->flags & RADEON_IS_IGP) {
4378                 mask |= GFX_POWER_GATING_ENABLE | GFX_POWER_GATING_SRC;
4379         }
4380
4381         WREG32(RLC_CNTL, mask);
4382 }
4383
4384 int evergreen_rlc_resume(struct radeon_device *rdev)
4385 {
4386         u32 i;
4387         const __be32 *fw_data;
4388
4389         if (!rdev->rlc_fw)
4390                 return -EINVAL;
4391
4392         r600_rlc_stop(rdev);
4393
4394         WREG32(RLC_HB_CNTL, 0);
4395
4396         if (rdev->flags & RADEON_IS_IGP) {
4397                 if (rdev->family == CHIP_ARUBA) {
4398                         u32 always_on_bitmap =
4399                                 3 | (3 << (16 * rdev->config.cayman.max_shader_engines));
4400                         /* find out the number of active simds */
4401                         u32 tmp = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
4402                         tmp |= 0xffffffff << rdev->config.cayman.max_simds_per_se;
4403                         tmp = hweight32(~tmp);
4404                         if (tmp == rdev->config.cayman.max_simds_per_se) {
4405                                 WREG32(TN_RLC_LB_ALWAYS_ACTIVE_SIMD_MASK, always_on_bitmap);
4406                                 WREG32(TN_RLC_LB_PARAMS, 0x00601004);
4407                                 WREG32(TN_RLC_LB_INIT_SIMD_MASK, 0xffffffff);
4408                                 WREG32(TN_RLC_LB_CNTR_INIT, 0x00000000);
4409                                 WREG32(TN_RLC_LB_CNTR_MAX, 0x00002000);
4410                         }
4411                 } else {
4412                         WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4413                         WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4414                 }
4415                 WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4416                 WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4417         } else {
4418                 WREG32(RLC_HB_BASE, 0);
4419                 WREG32(RLC_HB_RPTR, 0);
4420                 WREG32(RLC_HB_WPTR, 0);
4421                 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4422                 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4423         }
4424         WREG32(RLC_MC_CNTL, 0);
4425         WREG32(RLC_UCODE_CNTL, 0);
4426
4427         fw_data = (const __be32 *)rdev->rlc_fw->data;
4428         if (rdev->family >= CHIP_ARUBA) {
4429                 for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) {
4430                         WREG32(RLC_UCODE_ADDR, i);
4431                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4432                 }
4433         } else if (rdev->family >= CHIP_CAYMAN) {
4434                 for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) {
4435                         WREG32(RLC_UCODE_ADDR, i);
4436                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4437                 }
4438         } else {
4439                 for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
4440                         WREG32(RLC_UCODE_ADDR, i);
4441                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4442                 }
4443         }
4444         WREG32(RLC_UCODE_ADDR, 0);
4445
4446         evergreen_rlc_start(rdev);
4447
4448         return 0;
4449 }
4450
4451 /* Interrupts */
4452
4453 u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
4454 {
4455         if (crtc >= rdev->num_crtc)
4456                 return 0;
4457         else
4458                 return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
4459 }
4460
4461 void evergreen_disable_interrupt_state(struct radeon_device *rdev)
4462 {
4463         int i;
4464         u32 tmp;
4465
4466         if (rdev->family >= CHIP_CAYMAN) {
4467                 cayman_cp_int_cntl_setup(rdev, 0,
4468                                          CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4469                 cayman_cp_int_cntl_setup(rdev, 1, 0);
4470                 cayman_cp_int_cntl_setup(rdev, 2, 0);
4471                 tmp = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4472                 WREG32(CAYMAN_DMA1_CNTL, tmp);
4473         } else
4474                 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4475         tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4476         WREG32(DMA_CNTL, tmp);
4477         WREG32(GRBM_INT_CNTL, 0);
4478         WREG32(SRBM_INT_CNTL, 0);
4479         for (i = 0; i < rdev->num_crtc; i++)
4480                 WREG32(INT_MASK + crtc_offsets[i], 0);
4481         for (i = 0; i < rdev->num_crtc; i++)
4482                 WREG32(GRPH_INT_CONTROL + crtc_offsets[i], 0);
4483
4484         /* only one DAC on DCE5 */
4485         if (!ASIC_IS_DCE5(rdev))
4486                 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
4487         WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
4488
4489         for (i = 0; i < 6; i++)
4490                 WREG32_AND(DC_HPDx_INT_CONTROL(i), DC_HPDx_INT_POLARITY);
4491 }
4492
4493 /* Note that the order we write back regs here is important */
4494 int evergreen_irq_set(struct radeon_device *rdev)
4495 {
4496         int i;
4497         u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
4498         u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
4499         u32 grbm_int_cntl = 0;
4500         u32 dma_cntl, dma_cntl1 = 0;
4501         u32 thermal_int = 0;
4502
4503         if (!rdev->irq.installed) {
4504                 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
4505                 return -EINVAL;
4506         }
4507         /* don't enable anything if the ih is disabled */
4508         if (!rdev->ih.enabled) {
4509                 r600_disable_interrupts(rdev);
4510                 /* force the active interrupt state to all disabled */
4511                 evergreen_disable_interrupt_state(rdev);
4512                 return 0;
4513         }
4514
4515         if (rdev->family == CHIP_ARUBA)
4516                 thermal_int = RREG32(TN_CG_THERMAL_INT_CTRL) &
4517                         ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4518         else
4519                 thermal_int = RREG32(CG_THERMAL_INT) &
4520                         ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4521
4522         dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4523
4524         if (rdev->family >= CHIP_CAYMAN) {
4525                 /* enable CP interrupts on all rings */
4526                 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4527                         DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4528                         cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4529                 }
4530                 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
4531                         DRM_DEBUG("evergreen_irq_set: sw int cp1\n");
4532                         cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
4533                 }
4534                 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
4535                         DRM_DEBUG("evergreen_irq_set: sw int cp2\n");
4536                         cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
4537                 }
4538         } else {
4539                 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4540                         DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4541                         cp_int_cntl |= RB_INT_ENABLE;
4542                         cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4543                 }
4544         }
4545
4546         if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
4547                 DRM_DEBUG("r600_irq_set: sw int dma\n");
4548                 dma_cntl |= TRAP_ENABLE;
4549         }
4550
4551         if (rdev->family >= CHIP_CAYMAN) {
4552                 dma_cntl1 = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4553                 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
4554                         DRM_DEBUG("r600_irq_set: sw int dma1\n");
4555                         dma_cntl1 |= TRAP_ENABLE;
4556                 }
4557         }
4558
4559         if (rdev->irq.dpm_thermal) {
4560                 DRM_DEBUG("dpm thermal\n");
4561                 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
4562         }
4563
4564         if (rdev->family >= CHIP_CAYMAN) {
4565                 cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl);
4566                 cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1);
4567                 cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2);
4568         } else
4569                 WREG32(CP_INT_CNTL, cp_int_cntl);
4570
4571         WREG32(DMA_CNTL, dma_cntl);
4572
4573         if (rdev->family >= CHIP_CAYMAN)
4574                 WREG32(CAYMAN_DMA1_CNTL, dma_cntl1);
4575
4576         WREG32(GRBM_INT_CNTL, grbm_int_cntl);
4577
4578         for (i = 0; i < rdev->num_crtc; i++) {
4579                 radeon_irq_kms_set_irq_n_enabled(
4580                     rdev, INT_MASK + crtc_offsets[i],
4581                     VBLANK_INT_MASK,
4582                     rdev->irq.crtc_vblank_int[i] ||
4583                     atomic_read(&rdev->irq.pflip[i]), "vblank", i);
4584         }
4585
4586         for (i = 0; i < rdev->num_crtc; i++)
4587                 WREG32(GRPH_INT_CONTROL + crtc_offsets[i], GRPH_PFLIP_INT_MASK);
4588
4589         for (i = 0; i < 6; i++) {
4590                 radeon_irq_kms_set_irq_n_enabled(
4591                     rdev, DC_HPDx_INT_CONTROL(i),
4592                     DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN,
4593                     rdev->irq.hpd[i], "HPD", i);
4594         }
4595
4596         if (rdev->family == CHIP_ARUBA)
4597                 WREG32(TN_CG_THERMAL_INT_CTRL, thermal_int);
4598         else
4599                 WREG32(CG_THERMAL_INT, thermal_int);
4600
4601         for (i = 0; i < 6; i++) {
4602                 radeon_irq_kms_set_irq_n_enabled(
4603                     rdev, AFMT_AUDIO_PACKET_CONTROL + crtc_offsets[i],
4604                     AFMT_AZ_FORMAT_WTRIG_MASK,
4605                     rdev->irq.afmt[i], "HDMI", i);
4606         }
4607
4608         /* posting read */
4609         RREG32(SRBM_STATUS);
4610
4611         return 0;
4612 }
4613
4614 /* Note that the order we write back regs here is important */
4615 static void evergreen_irq_ack(struct radeon_device *rdev)
4616 {
4617         int i, j;
4618         u32 *grph_int = rdev->irq.stat_regs.evergreen.grph_int;
4619         u32 *disp_int = rdev->irq.stat_regs.evergreen.disp_int;
4620         u32 *afmt_status = rdev->irq.stat_regs.evergreen.afmt_status;
4621
4622         for (i = 0; i < 6; i++) {
4623                 disp_int[i] = RREG32(evergreen_disp_int_status[i]);
4624                 afmt_status[i] = RREG32(AFMT_STATUS + crtc_offsets[i]);
4625                 if (i < rdev->num_crtc)
4626                         grph_int[i] = RREG32(GRPH_INT_STATUS + crtc_offsets[i]);
4627         }
4628
4629         /* We write back each interrupt register in pairs of two */
4630         for (i = 0; i < rdev->num_crtc; i += 2) {
4631                 for (j = i; j < (i + 2); j++) {
4632                         if (grph_int[j] & GRPH_PFLIP_INT_OCCURRED)
4633                                 WREG32(GRPH_INT_STATUS + crtc_offsets[j],
4634                                        GRPH_PFLIP_INT_CLEAR);
4635                 }
4636
4637                 for (j = i; j < (i + 2); j++) {
4638                         if (disp_int[j] & LB_D1_VBLANK_INTERRUPT)
4639                                 WREG32(VBLANK_STATUS + crtc_offsets[j],
4640                                        VBLANK_ACK);
4641                         if (disp_int[j] & LB_D1_VLINE_INTERRUPT)
4642                                 WREG32(VLINE_STATUS + crtc_offsets[j],
4643                                        VLINE_ACK);
4644                 }
4645         }
4646
4647         for (i = 0; i < 6; i++) {
4648                 if (disp_int[i] & DC_HPD1_INTERRUPT)
4649                         WREG32_OR(DC_HPDx_INT_CONTROL(i), DC_HPDx_INT_ACK);
4650         }
4651
4652         for (i = 0; i < 6; i++) {
4653                 if (disp_int[i] & DC_HPD1_RX_INTERRUPT)
4654                         WREG32_OR(DC_HPDx_INT_CONTROL(i), DC_HPDx_RX_INT_ACK);
4655         }
4656
4657         for (i = 0; i < 6; i++) {
4658                 if (afmt_status[i] & AFMT_AZ_FORMAT_WTRIG)
4659                         WREG32_OR(AFMT_AUDIO_PACKET_CONTROL + crtc_offsets[i],
4660                                   AFMT_AZ_FORMAT_WTRIG_ACK);
4661         }
4662 }
4663
4664 static void evergreen_irq_disable(struct radeon_device *rdev)
4665 {
4666         r600_disable_interrupts(rdev);
4667         /* Wait and acknowledge irq */
4668         mdelay(1);
4669         evergreen_irq_ack(rdev);
4670         evergreen_disable_interrupt_state(rdev);
4671 }
4672
4673 void evergreen_irq_suspend(struct radeon_device *rdev)
4674 {
4675         evergreen_irq_disable(rdev);
4676         r600_rlc_stop(rdev);
4677 }
4678
4679 static u32 evergreen_get_ih_wptr(struct radeon_device *rdev)
4680 {
4681         u32 wptr, tmp;
4682
4683         if (rdev->wb.enabled)
4684                 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
4685         else
4686                 wptr = RREG32(IH_RB_WPTR);
4687
4688         if (wptr & RB_OVERFLOW) {
4689                 wptr &= ~RB_OVERFLOW;
4690                 /* When a ring buffer overflow happen start parsing interrupt
4691                  * from the last not overwritten vector (wptr + 16). Hopefully
4692                  * this should allow us to catchup.
4693                  */
4694                 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
4695                          wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
4696                 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
4697                 tmp = RREG32(IH_RB_CNTL);
4698                 tmp |= IH_WPTR_OVERFLOW_CLEAR;
4699                 WREG32(IH_RB_CNTL, tmp);
4700         }
4701         return (wptr & rdev->ih.ptr_mask);
4702 }
4703
4704 int evergreen_irq_process(struct radeon_device *rdev)
4705 {
4706         u32 *disp_int = rdev->irq.stat_regs.evergreen.disp_int;
4707         u32 *afmt_status = rdev->irq.stat_regs.evergreen.afmt_status;
4708         u32 crtc_idx, hpd_idx, afmt_idx;
4709         u32 mask;
4710         u32 wptr;
4711         u32 rptr;
4712         u32 src_id, src_data;
4713         u32 ring_index;
4714         bool queue_hotplug = false;
4715         bool queue_hdmi = false;
4716         bool queue_dp = false;
4717         bool queue_thermal = false;
4718         u32 status, addr;
4719         const char *event_name;
4720
4721         if (!rdev->ih.enabled || rdev->shutdown)
4722                 return IRQ_NONE;
4723
4724         wptr = evergreen_get_ih_wptr(rdev);
4725
4726 restart_ih:
4727         /* is somebody else already processing irqs? */
4728         if (atomic_xchg(&rdev->ih.lock, 1))
4729                 return IRQ_NONE;
4730
4731         rptr = rdev->ih.rptr;
4732         DRM_DEBUG("evergreen_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
4733
4734         /* Order reading of wptr vs. reading of IH ring data */
4735         rmb();
4736
4737         /* display interrupts */
4738         evergreen_irq_ack(rdev);
4739
4740         while (rptr != wptr) {
4741                 /* wptr/rptr are in bytes! */
4742                 ring_index = rptr / 4;
4743                 src_id =  le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
4744                 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
4745
4746                 switch (src_id) {
4747                 case 1: /* D1 vblank/vline */
4748                 case 2: /* D2 vblank/vline */
4749                 case 3: /* D3 vblank/vline */
4750                 case 4: /* D4 vblank/vline */
4751                 case 5: /* D5 vblank/vline */
4752                 case 6: /* D6 vblank/vline */
4753                         crtc_idx = src_id - 1;
4754
4755                         if (src_data == 0) { /* vblank */
4756                                 mask = LB_D1_VBLANK_INTERRUPT;
4757                                 event_name = "vblank";
4758
4759                                 if (rdev->irq.crtc_vblank_int[crtc_idx]) {
4760                                         drm_handle_vblank(rdev->ddev, crtc_idx);
4761                                         rdev->pm.vblank_sync = true;
4762                                         wake_up(&rdev->irq.vblank_queue);
4763                                 }
4764                                 if (atomic_read(&rdev->irq.pflip[crtc_idx])) {
4765                                         radeon_crtc_handle_vblank(rdev,
4766                                                                   crtc_idx);
4767                                 }
4768
4769                         } else if (src_data == 1) { /* vline */
4770                                 mask = LB_D1_VLINE_INTERRUPT;
4771                                 event_name = "vline";
4772                         } else {
4773                                 DRM_DEBUG("Unhandled interrupt: %d %d\n",
4774                                           src_id, src_data);
4775                                 break;
4776                         }
4777
4778                         if (!(disp_int[crtc_idx] & mask)) {
4779                                 DRM_DEBUG("IH: D%d %s - IH event w/o asserted irq bit?\n",
4780                                           crtc_idx + 1, event_name);
4781                         }
4782
4783                         disp_int[crtc_idx] &= ~mask;
4784                         DRM_DEBUG("IH: D%d %s\n", crtc_idx + 1, event_name);
4785
4786                         break;
4787                 case 8: /* D1 page flip */
4788                 case 10: /* D2 page flip */
4789                 case 12: /* D3 page flip */
4790                 case 14: /* D4 page flip */
4791                 case 16: /* D5 page flip */
4792                 case 18: /* D6 page flip */
4793                         DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
4794                         if (radeon_use_pflipirq > 0)
4795                                 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
4796                         break;
4797                 case 42: /* HPD hotplug */
4798                         if (src_data <= 5) {
4799                                 hpd_idx = src_data;
4800                                 mask = DC_HPD1_INTERRUPT;
4801                                 queue_hotplug = true;
4802                                 event_name = "HPD";
4803
4804                         } else if (src_data <= 11) {
4805                                 hpd_idx = src_data - 6;
4806                                 mask = DC_HPD1_RX_INTERRUPT;
4807                                 queue_dp = true;
4808                                 event_name = "HPD_RX";
4809
4810                         } else {
4811                                 DRM_DEBUG("Unhandled interrupt: %d %d\n",
4812                                           src_id, src_data);
4813                                 break;
4814                         }
4815
4816                         if (!(disp_int[hpd_idx] & mask))
4817                                 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
4818
4819                         disp_int[hpd_idx] &= ~mask;
4820                         DRM_DEBUG("IH: %s%d\n", event_name, hpd_idx + 1);
4821
4822                         break;
4823                 case 44: /* hdmi */
4824                         afmt_idx = src_data;
4825                         if (afmt_idx > 5) {
4826                                 DRM_ERROR("Unhandled interrupt: %d %d\n",
4827                                           src_id, src_data);
4828                                 break;
4829                         }
4830
4831                         if (!(afmt_status[afmt_idx] & AFMT_AZ_FORMAT_WTRIG))
4832                                 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
4833
4834                         afmt_status[afmt_idx] &= ~AFMT_AZ_FORMAT_WTRIG;
4835                         queue_hdmi = true;
4836                         DRM_DEBUG("IH: HDMI%d\n", afmt_idx + 1);
4837                         break;
4838                 case 96:
4839                         DRM_ERROR("SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR));
4840                         WREG32(SRBM_INT_ACK, 0x1);
4841                         break;
4842                 case 124: /* UVD */
4843                         DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
4844                         radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
4845                         break;
4846                 case 146:
4847                 case 147:
4848                         addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
4849                         status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
4850                         /* reset addr and status */
4851                         WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
4852                         if (addr == 0x0 && status == 0x0)
4853                                 break;
4854                         dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
4855                         dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
4856                                 addr);
4857                         dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4858                                 status);
4859                         cayman_vm_decode_fault(rdev, status, addr);
4860                         break;
4861                 case 176: /* CP_INT in ring buffer */
4862                 case 177: /* CP_INT in IB1 */
4863                 case 178: /* CP_INT in IB2 */
4864                         DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
4865                         radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4866                         break;
4867                 case 181: /* CP EOP event */
4868                         DRM_DEBUG("IH: CP EOP\n");
4869                         if (rdev->family >= CHIP_CAYMAN) {
4870                                 switch (src_data) {
4871                                 case 0:
4872                                         radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4873                                         break;
4874                                 case 1:
4875                                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
4876                                         break;
4877                                 case 2:
4878                                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
4879                                         break;
4880                                 }
4881                         } else
4882                                 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4883                         break;
4884                 case 224: /* DMA trap event */
4885                         DRM_DEBUG("IH: DMA trap\n");
4886                         radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
4887                         break;
4888                 case 230: /* thermal low to high */
4889                         DRM_DEBUG("IH: thermal low to high\n");
4890                         rdev->pm.dpm.thermal.high_to_low = false;
4891                         queue_thermal = true;
4892                         break;
4893                 case 231: /* thermal high to low */
4894                         DRM_DEBUG("IH: thermal high to low\n");
4895                         rdev->pm.dpm.thermal.high_to_low = true;
4896                         queue_thermal = true;
4897                         break;
4898                 case 233: /* GUI IDLE */
4899                         DRM_DEBUG("IH: GUI idle\n");
4900                         break;
4901                 case 244: /* DMA trap event */
4902                         if (rdev->family >= CHIP_CAYMAN) {
4903                                 DRM_DEBUG("IH: DMA1 trap\n");
4904                                 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
4905                         }
4906                         break;
4907                 default:
4908                         DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4909                         break;
4910                 }
4911
4912                 /* wptr/rptr are in bytes! */
4913                 rptr += 16;
4914                 rptr &= rdev->ih.ptr_mask;
4915                 WREG32(IH_RB_RPTR, rptr);
4916         }
4917         if (queue_dp)
4918                 schedule_work(&rdev->dp_work);
4919         if (queue_hotplug)
4920                 schedule_delayed_work(&rdev->hotplug_work, 0);
4921         if (queue_hdmi)
4922                 schedule_work(&rdev->audio_work);
4923         if (queue_thermal && rdev->pm.dpm_enabled)
4924                 schedule_work(&rdev->pm.dpm.thermal.work);
4925         rdev->ih.rptr = rptr;
4926         atomic_set(&rdev->ih.lock, 0);
4927
4928         /* make sure wptr hasn't changed while processing */
4929         wptr = evergreen_get_ih_wptr(rdev);
4930         if (wptr != rptr)
4931                 goto restart_ih;
4932
4933         return IRQ_HANDLED;
4934 }
4935
4936 static void evergreen_uvd_init(struct radeon_device *rdev)
4937 {
4938         int r;
4939
4940         if (!rdev->has_uvd)
4941                 return;
4942
4943         r = radeon_uvd_init(rdev);
4944         if (r) {
4945                 dev_err(rdev->dev, "failed UVD (%d) init.\n", r);
4946                 /*
4947                  * At this point rdev->uvd.vcpu_bo is NULL which trickles down
4948                  * to early fails uvd_v2_2_resume() and thus nothing happens
4949                  * there. So it is pointless to try to go through that code
4950                  * hence why we disable uvd here.
4951                  */
4952                 rdev->has_uvd = false;
4953                 return;
4954         }
4955         rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
4956         r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX], 4096);
4957 }
4958
4959 static void evergreen_uvd_start(struct radeon_device *rdev)
4960 {
4961         int r;
4962
4963         if (!rdev->has_uvd)
4964                 return;
4965
4966         r = uvd_v2_2_resume(rdev);
4967         if (r) {
4968                 dev_err(rdev->dev, "failed UVD resume (%d).\n", r);
4969                 goto error;
4970         }
4971         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_UVD_INDEX);
4972         if (r) {
4973                 dev_err(rdev->dev, "failed initializing UVD fences (%d).\n", r);
4974                 goto error;
4975         }
4976         return;
4977
4978 error:
4979         rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
4980 }
4981
4982 static void evergreen_uvd_resume(struct radeon_device *rdev)
4983 {
4984         struct radeon_ring *ring;
4985         int r;
4986
4987         if (!rdev->has_uvd || !rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size)
4988                 return;
4989
4990         ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
4991         r = radeon_ring_init(rdev, ring, ring->ring_size, 0, PACKET0(UVD_NO_OP, 0));
4992         if (r) {
4993                 dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r);
4994                 return;
4995         }
4996         r = uvd_v1_0_init(rdev);
4997         if (r) {
4998                 dev_err(rdev->dev, "failed initializing UVD (%d).\n", r);
4999                 return;
5000         }
5001 }
5002
5003 static int evergreen_startup(struct radeon_device *rdev)
5004 {
5005         struct radeon_ring *ring;
5006         int r;
5007
5008         /* enable pcie gen2 link */
5009         evergreen_pcie_gen2_enable(rdev);
5010         /* enable aspm */
5011         evergreen_program_aspm(rdev);
5012
5013         /* scratch needs to be initialized before MC */
5014         r = r600_vram_scratch_init(rdev);
5015         if (r)
5016                 return r;
5017
5018         evergreen_mc_program(rdev);
5019
5020         if (ASIC_IS_DCE5(rdev) && !rdev->pm.dpm_enabled) {
5021                 r = ni_mc_load_microcode(rdev);
5022                 if (r) {
5023                         DRM_ERROR("Failed to load MC firmware!\n");
5024                         return r;
5025                 }
5026         }
5027
5028         if (rdev->flags & RADEON_IS_AGP) {
5029                 evergreen_agp_enable(rdev);
5030         } else {
5031                 r = evergreen_pcie_gart_enable(rdev);
5032                 if (r)
5033                         return r;
5034         }
5035         evergreen_gpu_init(rdev);
5036
5037         /* allocate rlc buffers */
5038         if (rdev->flags & RADEON_IS_IGP) {
5039                 rdev->rlc.reg_list = sumo_rlc_save_restore_register_list;
5040                 rdev->rlc.reg_list_size =
5041                         (u32)ARRAY_SIZE(sumo_rlc_save_restore_register_list);
5042                 rdev->rlc.cs_data = evergreen_cs_data;
5043                 r = sumo_rlc_init(rdev);
5044                 if (r) {
5045                         DRM_ERROR("Failed to init rlc BOs!\n");
5046                         return r;
5047                 }
5048         }
5049
5050         /* allocate wb buffer */
5051         r = radeon_wb_init(rdev);
5052         if (r)
5053                 return r;
5054
5055         r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
5056         if (r) {
5057                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5058                 return r;
5059         }
5060
5061         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
5062         if (r) {
5063                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
5064                 return r;
5065         }
5066
5067         evergreen_uvd_start(rdev);
5068
5069         /* Enable IRQ */
5070         if (!rdev->irq.installed) {
5071                 r = radeon_irq_kms_init(rdev);
5072                 if (r)
5073                         return r;
5074         }
5075
5076         r = r600_irq_init(rdev);
5077         if (r) {
5078                 DRM_ERROR("radeon: IH init failed (%d).\n", r);
5079                 radeon_irq_kms_fini(rdev);
5080                 return r;
5081         }
5082         evergreen_irq_set(rdev);
5083
5084         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
5085         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
5086                              RADEON_CP_PACKET2);
5087         if (r)
5088                 return r;
5089
5090         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
5091         r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
5092                              DMA_PACKET(DMA_PACKET_NOP, 0, 0));
5093         if (r)
5094                 return r;
5095
5096         r = evergreen_cp_load_microcode(rdev);
5097         if (r)
5098                 return r;
5099         r = evergreen_cp_resume(rdev);
5100         if (r)
5101                 return r;
5102         r = r600_dma_resume(rdev);
5103         if (r)
5104                 return r;
5105
5106         evergreen_uvd_resume(rdev);
5107
5108         r = radeon_ib_pool_init(rdev);
5109         if (r) {
5110                 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
5111                 return r;
5112         }
5113
5114         r = radeon_audio_init(rdev);
5115         if (r) {
5116                 DRM_ERROR("radeon: audio init failed\n");
5117                 return r;
5118         }
5119
5120         return 0;
5121 }
5122
5123 int evergreen_resume(struct radeon_device *rdev)
5124 {
5125         int r;
5126
5127         /* reset the asic, the gfx blocks are often in a bad state
5128          * after the driver is unloaded or after a resume
5129          */
5130         if (radeon_asic_reset(rdev))
5131                 dev_warn(rdev->dev, "GPU reset failed !\n");
5132         /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
5133          * posting will perform necessary task to bring back GPU into good
5134          * shape.
5135          */
5136         /* post card */
5137         atom_asic_init(rdev->mode_info.atom_context);
5138
5139         /* init golden registers */
5140         evergreen_init_golden_registers(rdev);
5141
5142         if (rdev->pm.pm_method == PM_METHOD_DPM)
5143                 radeon_pm_resume(rdev);
5144
5145         rdev->accel_working = true;
5146         r = evergreen_startup(rdev);
5147         if (r) {
5148                 DRM_ERROR("evergreen startup failed on resume\n");
5149                 rdev->accel_working = false;
5150                 return r;
5151         }
5152
5153         return r;
5154
5155 }
5156
5157 int evergreen_suspend(struct radeon_device *rdev)
5158 {
5159         radeon_pm_suspend(rdev);
5160         radeon_audio_fini(rdev);
5161         if (rdev->has_uvd) {
5162                 radeon_uvd_suspend(rdev);
5163                 uvd_v1_0_fini(rdev);
5164         }
5165         r700_cp_stop(rdev);
5166         r600_dma_stop(rdev);
5167         evergreen_irq_suspend(rdev);
5168         radeon_wb_disable(rdev);
5169         evergreen_pcie_gart_disable(rdev);
5170
5171         return 0;
5172 }
5173
5174 /* Plan is to move initialization in that function and use
5175  * helper function so that radeon_device_init pretty much
5176  * do nothing more than calling asic specific function. This
5177  * should also allow to remove a bunch of callback function
5178  * like vram_info.
5179  */
5180 int evergreen_init(struct radeon_device *rdev)
5181 {
5182         int r;
5183
5184         /* Read BIOS */
5185         if (!radeon_get_bios(rdev)) {
5186                 if (ASIC_IS_AVIVO(rdev))
5187                         return -EINVAL;
5188         }
5189         /* Must be an ATOMBIOS */
5190         if (!rdev->is_atom_bios) {
5191                 dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
5192                 return -EINVAL;
5193         }
5194         r = radeon_atombios_init(rdev);
5195         if (r)
5196                 return r;
5197         /* reset the asic, the gfx blocks are often in a bad state
5198          * after the driver is unloaded or after a resume
5199          */
5200         if (radeon_asic_reset(rdev))
5201                 dev_warn(rdev->dev, "GPU reset failed !\n");
5202         /* Post card if necessary */
5203         if (!radeon_card_posted(rdev)) {
5204                 if (!rdev->bios) {
5205                         dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
5206                         return -EINVAL;
5207                 }
5208                 DRM_INFO("GPU not posted. posting now...\n");
5209                 atom_asic_init(rdev->mode_info.atom_context);
5210         }
5211         /* init golden registers */
5212         evergreen_init_golden_registers(rdev);
5213         /* Initialize scratch registers */
5214         r600_scratch_init(rdev);
5215         /* Initialize surface registers */
5216         radeon_surface_init(rdev);
5217         /* Initialize clocks */
5218         radeon_get_clock_info(rdev->ddev);
5219         /* Fence driver */
5220         radeon_fence_driver_init(rdev);
5221         /* initialize AGP */
5222         if (rdev->flags & RADEON_IS_AGP) {
5223                 r = radeon_agp_init(rdev);
5224                 if (r)
5225                         radeon_agp_disable(rdev);
5226         }
5227         /* initialize memory controller */
5228         r = evergreen_mc_init(rdev);
5229         if (r)
5230                 return r;
5231         /* Memory manager */
5232         r = radeon_bo_init(rdev);
5233         if (r)
5234                 return r;
5235
5236         if (ASIC_IS_DCE5(rdev)) {
5237                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
5238                         r = ni_init_microcode(rdev);
5239                         if (r) {
5240                                 DRM_ERROR("Failed to load firmware!\n");
5241                                 return r;
5242                         }
5243                 }
5244         } else {
5245                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
5246                         r = r600_init_microcode(rdev);
5247                         if (r) {
5248                                 DRM_ERROR("Failed to load firmware!\n");
5249                                 return r;
5250                         }
5251                 }
5252         }
5253
5254         /* Initialize power management */
5255         radeon_pm_init(rdev);
5256
5257         rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
5258         r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
5259
5260         rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
5261         r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
5262
5263         evergreen_uvd_init(rdev);
5264
5265         rdev->ih.ring_obj = NULL;
5266         r600_ih_ring_init(rdev, 64 * 1024);
5267
5268         r = r600_pcie_gart_init(rdev);
5269         if (r)
5270                 return r;
5271
5272         rdev->accel_working = true;
5273         r = evergreen_startup(rdev);
5274         if (r) {
5275                 dev_err(rdev->dev, "disabling GPU acceleration\n");
5276                 r700_cp_fini(rdev);
5277                 r600_dma_fini(rdev);
5278                 r600_irq_fini(rdev);
5279                 if (rdev->flags & RADEON_IS_IGP)
5280                         sumo_rlc_fini(rdev);
5281                 radeon_wb_fini(rdev);
5282                 radeon_ib_pool_fini(rdev);
5283                 radeon_irq_kms_fini(rdev);
5284                 evergreen_pcie_gart_fini(rdev);
5285                 rdev->accel_working = false;
5286         }
5287
5288         /* Don't start up if the MC ucode is missing on BTC parts.
5289          * The default clocks and voltages before the MC ucode
5290          * is loaded are not suffient for advanced operations.
5291          */
5292         if (ASIC_IS_DCE5(rdev)) {
5293                 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
5294                         DRM_ERROR("radeon: MC ucode required for NI+.\n");
5295                         return -EINVAL;
5296                 }
5297         }
5298
5299         return 0;
5300 }
5301
5302 void evergreen_fini(struct radeon_device *rdev)
5303 {
5304         radeon_pm_fini(rdev);
5305         radeon_audio_fini(rdev);
5306         r700_cp_fini(rdev);
5307         r600_dma_fini(rdev);
5308         r600_irq_fini(rdev);
5309         if (rdev->flags & RADEON_IS_IGP)
5310                 sumo_rlc_fini(rdev);
5311         radeon_wb_fini(rdev);
5312         radeon_ib_pool_fini(rdev);
5313         radeon_irq_kms_fini(rdev);
5314         uvd_v1_0_fini(rdev);
5315         radeon_uvd_fini(rdev);
5316         evergreen_pcie_gart_fini(rdev);
5317         r600_vram_scratch_fini(rdev);
5318         radeon_gem_fini(rdev);
5319         radeon_fence_driver_fini(rdev);
5320         radeon_agp_fini(rdev);
5321         radeon_bo_fini(rdev);
5322         radeon_atombios_fini(rdev);
5323         kfree(rdev->bios);
5324         rdev->bios = NULL;
5325 }
5326
5327 void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
5328 {
5329         u32 link_width_cntl, speed_cntl;
5330
5331         if (radeon_pcie_gen2 == 0)
5332                 return;
5333
5334         if (rdev->flags & RADEON_IS_IGP)
5335                 return;
5336
5337         if (!(rdev->flags & RADEON_IS_PCIE))
5338                 return;
5339
5340         /* x2 cards have a special sequence */
5341         if (ASIC_IS_X2(rdev))
5342                 return;
5343
5344         if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
5345                 (rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
5346                 return;
5347
5348         speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5349         if (speed_cntl & LC_CURRENT_DATA_RATE) {
5350                 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
5351                 return;
5352         }
5353
5354         DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
5355
5356         if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
5357             (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
5358
5359                 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5360                 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5361                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5362
5363                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5364                 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
5365                 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5366
5367                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5368                 speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
5369                 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5370
5371                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5372                 speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
5373                 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5374
5375                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5376                 speed_cntl |= LC_GEN2_EN_STRAP;
5377                 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5378
5379         } else {
5380                 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5381                 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
5382                 if (1)
5383                         link_width_cntl |= LC_UPCONFIGURE_DIS;
5384                 else
5385                         link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5386                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5387         }
5388 }
5389
5390 void evergreen_program_aspm(struct radeon_device *rdev)
5391 {
5392         u32 data, orig;
5393         u32 pcie_lc_cntl, pcie_lc_cntl_old;
5394         bool disable_l0s, disable_l1 = false, disable_plloff_in_l1 = false;
5395         /* fusion_platform = true
5396          * if the system is a fusion system
5397          * (APU or DGPU in a fusion system).
5398          * todo: check if the system is a fusion platform.
5399          */
5400         bool fusion_platform = false;
5401
5402         if (radeon_aspm == 0)
5403                 return;
5404
5405         if (!(rdev->flags & RADEON_IS_PCIE))
5406                 return;
5407
5408         switch (rdev->family) {
5409         case CHIP_CYPRESS:
5410         case CHIP_HEMLOCK:
5411         case CHIP_JUNIPER:
5412         case CHIP_REDWOOD:
5413         case CHIP_CEDAR:
5414         case CHIP_SUMO:
5415         case CHIP_SUMO2:
5416         case CHIP_PALM:
5417         case CHIP_ARUBA:
5418                 disable_l0s = true;
5419                 break;
5420         default:
5421                 disable_l0s = false;
5422                 break;
5423         }
5424
5425         if (rdev->flags & RADEON_IS_IGP)
5426                 fusion_platform = true; /* XXX also dGPUs in a fusion system */
5427
5428         data = orig = RREG32_PIF_PHY0(PB0_PIF_PAIRING);
5429         if (fusion_platform)
5430                 data &= ~MULTI_PIF;
5431         else
5432                 data |= MULTI_PIF;
5433         if (data != orig)
5434                 WREG32_PIF_PHY0(PB0_PIF_PAIRING, data);
5435
5436         data = orig = RREG32_PIF_PHY1(PB1_PIF_PAIRING);
5437         if (fusion_platform)
5438                 data &= ~MULTI_PIF;
5439         else
5440                 data |= MULTI_PIF;
5441         if (data != orig)
5442                 WREG32_PIF_PHY1(PB1_PIF_PAIRING, data);
5443
5444         pcie_lc_cntl = pcie_lc_cntl_old = RREG32_PCIE_PORT(PCIE_LC_CNTL);
5445         pcie_lc_cntl &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
5446         if (!disable_l0s) {
5447                 if (rdev->family >= CHIP_BARTS)
5448                         pcie_lc_cntl |= LC_L0S_INACTIVITY(7);
5449                 else
5450                         pcie_lc_cntl |= LC_L0S_INACTIVITY(3);
5451         }
5452
5453         if (!disable_l1) {
5454                 if (rdev->family >= CHIP_BARTS)
5455                         pcie_lc_cntl |= LC_L1_INACTIVITY(7);
5456                 else
5457                         pcie_lc_cntl |= LC_L1_INACTIVITY(8);
5458
5459                 if (!disable_plloff_in_l1) {
5460                         data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5461                         data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5462                         data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5463                         if (data != orig)
5464                                 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5465
5466                         data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5467                         data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5468                         data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5469                         if (data != orig)
5470                                 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5471
5472                         data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5473                         data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5474                         data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5475                         if (data != orig)
5476                                 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5477
5478                         data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5479                         data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5480                         data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5481                         if (data != orig)
5482                                 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5483
5484                         if (rdev->family >= CHIP_BARTS) {
5485                                 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5486                                 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5487                                 data |= PLL_RAMP_UP_TIME_0(4);
5488                                 if (data != orig)
5489                                         WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5490
5491                                 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5492                                 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5493                                 data |= PLL_RAMP_UP_TIME_1(4);
5494                                 if (data != orig)
5495                                         WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5496
5497                                 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5498                                 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5499                                 data |= PLL_RAMP_UP_TIME_0(4);
5500                                 if (data != orig)
5501                                         WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5502
5503                                 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5504                                 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5505                                 data |= PLL_RAMP_UP_TIME_1(4);
5506                                 if (data != orig)
5507                                         WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5508                         }
5509
5510                         data = orig = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5511                         data &= ~LC_DYN_LANES_PWR_STATE_MASK;
5512                         data |= LC_DYN_LANES_PWR_STATE(3);
5513                         if (data != orig)
5514                                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
5515
5516                         if (rdev->family >= CHIP_BARTS) {
5517                                 data = orig = RREG32_PIF_PHY0(PB0_PIF_CNTL);
5518                                 data &= ~LS2_EXIT_TIME_MASK;
5519                                 data |= LS2_EXIT_TIME(1);
5520                                 if (data != orig)
5521                                         WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
5522
5523                                 data = orig = RREG32_PIF_PHY1(PB1_PIF_CNTL);
5524                                 data &= ~LS2_EXIT_TIME_MASK;
5525                                 data |= LS2_EXIT_TIME(1);
5526                                 if (data != orig)
5527                                         WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
5528                         }
5529                 }
5530         }
5531
5532         /* evergreen parts only */
5533         if (rdev->family < CHIP_BARTS)
5534                 pcie_lc_cntl |= LC_PMI_TO_L1_DIS;
5535
5536         if (pcie_lc_cntl != pcie_lc_cntl_old)
5537                 WREG32_PCIE_PORT(PCIE_LC_CNTL, pcie_lc_cntl);
5538 }