drivers/gpu/drm/radeon/rv770.c

   1 /*
   2  * Copyright 2008 Advanced Micro Devices, Inc.
   3  * Copyright 2008 Red Hat Inc.
   4  * Copyright 2009 Jerome Glisse.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22  * OTHER DEALINGS IN THE SOFTWARE.
  23  *
  24  * Authors: Dave Airlie
  25  *          Alex Deucher
  26  *          Jerome Glisse
  27  */
  28
  29 #include <linux/firmware.h>
  30 #include <linux/pci.h>
  31 #include <linux/slab.h>
  32
  33 #include <drm/drm_device.h>
  34 #include <drm/radeon_drm.h>
  35
  36 #include "atom.h"
  37 #include "avivod.h"
  38 #include "radeon.h"
  39 #include "radeon_asic.h"
  40 #include "radeon_audio.h"
  41 #include "rv770d.h"
  42 #include "rv770.h"
  43
  44 #define R700_PFP_UCODE_SIZE 848
  45 #define R700_PM4_UCODE_SIZE 1360
  46
  47 static void rv770_gpu_init(struct radeon_device *rdev);
  48 void rv770_fini(struct radeon_device *rdev);
  49 static void rv770_pcie_gen2_enable(struct radeon_device *rdev);
  50 int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk);
  51
  52 int rv770_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
  53 {
  54         unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
  55         int r;
  56
  57         /* RV740 uses evergreen uvd clk programming */
  58         if (rdev->family == CHIP_RV740)
  59                 return evergreen_set_uvd_clocks(rdev, vclk, dclk);
  60
  61         /* bypass vclk and dclk with bclk */
  62         WREG32_P(CG_UPLL_FUNC_CNTL_2,
  63                  VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
  64                  ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
  65
  66         if (!vclk || !dclk) {
  67                 /* keep the Bypass mode, put PLL to sleep */
  68                 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
  69                 return 0;
  70         }
  71
  72         r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 50000, 160000,
  73                                           43663, 0x03FFFFFE, 1, 30, ~0,
  74                                           &fb_div, &vclk_div, &dclk_div);
  75         if (r)
  76                 return r;
  77
  78         fb_div |= 1;
  79         vclk_div -= 1;
  80         dclk_div -= 1;
  81
  82         /* set UPLL_FB_DIV to 0x50000 */
  83         WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(0x50000), ~UPLL_FB_DIV_MASK);
  84
  85         /* deassert UPLL_RESET and UPLL_SLEEP */
  86         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~(UPLL_RESET_MASK | UPLL_SLEEP_MASK));
  87
  88         /* assert BYPASS EN and FB_DIV[0] <- ??? why? */
  89         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
  90         WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(1), ~UPLL_FB_DIV(1));
  91
  92         r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
  93         if (r)
  94                 return r;
  95
  96         /* assert PLL_RESET */
  97         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
  98
  99         /* set the required FB_DIV, REF_DIV, Post divder values */
 100         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_REF_DIV(1), ~UPLL_REF_DIV_MASK);
 101         WREG32_P(CG_UPLL_FUNC_CNTL_2,
 102                  UPLL_SW_HILEN(vclk_div >> 1) |
 103                  UPLL_SW_LOLEN((vclk_div >> 1) + (vclk_div & 1)) |
 104                  UPLL_SW_HILEN2(dclk_div >> 1) |
 105                  UPLL_SW_LOLEN2((dclk_div >> 1) + (dclk_div & 1)),
 106                  ~UPLL_SW_MASK);
 107
 108         WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div),
 109                  ~UPLL_FB_DIV_MASK);
 110
 111         /* give the PLL some time to settle */
 112         mdelay(15);
 113
 114         /* deassert PLL_RESET */
 115         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
 116
 117         mdelay(15);
 118
 119         /* deassert BYPASS EN and FB_DIV[0] <- ??? why? */
 120         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
 121         WREG32_P(CG_UPLL_FUNC_CNTL_3, 0, ~UPLL_FB_DIV(1));
 122
 123         r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
 124         if (r)
 125                 return r;
 126
 127         /* switch VCLK and DCLK selection */
 128         WREG32_P(CG_UPLL_FUNC_CNTL_2,
 129                  VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
 130                  ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
 131
 132         mdelay(100);
 133
 134         return 0;
 135 }
 136
 137 static const u32 r7xx_golden_registers[] =
 138 {
 139         0x8d00, 0xffffffff, 0x0e0e0074,
 140         0x8d04, 0xffffffff, 0x013a2b34,
 141         0x9508, 0xffffffff, 0x00000002,
 142         0x8b20, 0xffffffff, 0,
 143         0x88c4, 0xffffffff, 0x000000c2,
 144         0x28350, 0xffffffff, 0,
 145         0x9058, 0xffffffff, 0x0fffc40f,
 146         0x240c, 0xffffffff, 0x00000380,
 147         0x733c, 0xffffffff, 0x00000002,
 148         0x2650, 0x00040000, 0,
 149         0x20bc, 0x00040000, 0,
 150         0x7300, 0xffffffff, 0x001000f0
 151 };
 152
 153 static const u32 r7xx_golden_dyn_gpr_registers[] =
 154 {
 155         0x8db0, 0xffffffff, 0x98989898,
 156         0x8db4, 0xffffffff, 0x98989898,
 157         0x8db8, 0xffffffff, 0x98989898,
 158         0x8dbc, 0xffffffff, 0x98989898,
 159         0x8dc0, 0xffffffff, 0x98989898,
 160         0x8dc4, 0xffffffff, 0x98989898,
 161         0x8dc8, 0xffffffff, 0x98989898,
 162         0x8dcc, 0xffffffff, 0x98989898,
 163         0x88c4, 0xffffffff, 0x00000082
 164 };
 165
 166 static const u32 rv770_golden_registers[] =
 167 {
 168         0x562c, 0xffffffff, 0,
 169         0x3f90, 0xffffffff, 0,
 170         0x9148, 0xffffffff, 0,
 171         0x3f94, 0xffffffff, 0,
 172         0x914c, 0xffffffff, 0,
 173         0x9698, 0x18000000, 0x18000000
 174 };
 175
 176 static const u32 rv770ce_golden_registers[] =
 177 {
 178         0x562c, 0xffffffff, 0,
 179         0x3f90, 0xffffffff, 0x00cc0000,
 180         0x9148, 0xffffffff, 0x00cc0000,
 181         0x3f94, 0xffffffff, 0x00cc0000,
 182         0x914c, 0xffffffff, 0x00cc0000,
 183         0x9b7c, 0xffffffff, 0x00fa0000,
 184         0x3f8c, 0xffffffff, 0x00fa0000,
 185         0x9698, 0x18000000, 0x18000000
 186 };
 187
 188 static const u32 rv770_mgcg_init[] =
 189 {
 190         0x8bcc, 0xffffffff, 0x130300f9,
 191         0x5448, 0xffffffff, 0x100,
 192         0x55e4, 0xffffffff, 0x100,
 193         0x160c, 0xffffffff, 0x100,
 194         0x5644, 0xffffffff, 0x100,
 195         0xc164, 0xffffffff, 0x100,
 196         0x8a18, 0xffffffff, 0x100,
 197         0x897c, 0xffffffff, 0x8000100,
 198         0x8b28, 0xffffffff, 0x3c000100,
 199         0x9144, 0xffffffff, 0x100,
 200         0x9a1c, 0xffffffff, 0x10000,
 201         0x9a50, 0xffffffff, 0x100,
 202         0x9a1c, 0xffffffff, 0x10001,
 203         0x9a50, 0xffffffff, 0x100,
 204         0x9a1c, 0xffffffff, 0x10002,
 205         0x9a50, 0xffffffff, 0x100,
 206         0x9a1c, 0xffffffff, 0x10003,
 207         0x9a50, 0xffffffff, 0x100,
 208         0x9a1c, 0xffffffff, 0x0,
 209         0x9870, 0xffffffff, 0x100,
 210         0x8d58, 0xffffffff, 0x100,
 211         0x9500, 0xffffffff, 0x0,
 212         0x9510, 0xffffffff, 0x100,
 213         0x9500, 0xffffffff, 0x1,
 214         0x9510, 0xffffffff, 0x100,
 215         0x9500, 0xffffffff, 0x2,
 216         0x9510, 0xffffffff, 0x100,
 217         0x9500, 0xffffffff, 0x3,
 218         0x9510, 0xffffffff, 0x100,
 219         0x9500, 0xffffffff, 0x4,
 220         0x9510, 0xffffffff, 0x100,
 221         0x9500, 0xffffffff, 0x5,
 222         0x9510, 0xffffffff, 0x100,
 223         0x9500, 0xffffffff, 0x6,
 224         0x9510, 0xffffffff, 0x100,
 225         0x9500, 0xffffffff, 0x7,
 226         0x9510, 0xffffffff, 0x100,
 227         0x9500, 0xffffffff, 0x8,
 228         0x9510, 0xffffffff, 0x100,
 229         0x9500, 0xffffffff, 0x9,
 230         0x9510, 0xffffffff, 0x100,
 231         0x9500, 0xffffffff, 0x8000,
 232         0x9490, 0xffffffff, 0x0,
 233         0x949c, 0xffffffff, 0x100,
 234         0x9490, 0xffffffff, 0x1,
 235         0x949c, 0xffffffff, 0x100,
 236         0x9490, 0xffffffff, 0x2,
 237         0x949c, 0xffffffff, 0x100,
 238         0x9490, 0xffffffff, 0x3,
 239         0x949c, 0xffffffff, 0x100,
 240         0x9490, 0xffffffff, 0x4,
 241         0x949c, 0xffffffff, 0x100,
 242         0x9490, 0xffffffff, 0x5,
 243         0x949c, 0xffffffff, 0x100,
 244         0x9490, 0xffffffff, 0x6,
 245         0x949c, 0xffffffff, 0x100,
 246         0x9490, 0xffffffff, 0x7,
 247         0x949c, 0xffffffff, 0x100,
 248         0x9490, 0xffffffff, 0x8,
 249         0x949c, 0xffffffff, 0x100,
 250         0x9490, 0xffffffff, 0x9,
 251         0x949c, 0xffffffff, 0x100,
 252         0x9490, 0xffffffff, 0x8000,
 253         0x9604, 0xffffffff, 0x0,
 254         0x9654, 0xffffffff, 0x100,
 255         0x9604, 0xffffffff, 0x1,
 256         0x9654, 0xffffffff, 0x100,
 257         0x9604, 0xffffffff, 0x2,
 258         0x9654, 0xffffffff, 0x100,
 259         0x9604, 0xffffffff, 0x3,
 260         0x9654, 0xffffffff, 0x100,
 261         0x9604, 0xffffffff, 0x4,
 262         0x9654, 0xffffffff, 0x100,
 263         0x9604, 0xffffffff, 0x5,
 264         0x9654, 0xffffffff, 0x100,
 265         0x9604, 0xffffffff, 0x6,
 266         0x9654, 0xffffffff, 0x100,
 267         0x9604, 0xffffffff, 0x7,
 268         0x9654, 0xffffffff, 0x100,
 269         0x9604, 0xffffffff, 0x8,
 270         0x9654, 0xffffffff, 0x100,
 271         0x9604, 0xffffffff, 0x9,
 272         0x9654, 0xffffffff, 0x100,
 273         0x9604, 0xffffffff, 0x80000000,
 274         0x9030, 0xffffffff, 0x100,
 275         0x9034, 0xffffffff, 0x100,
 276         0x9038, 0xffffffff, 0x100,
 277         0x903c, 0xffffffff, 0x100,
 278         0x9040, 0xffffffff, 0x100,
 279         0xa200, 0xffffffff, 0x100,
 280         0xa204, 0xffffffff, 0x100,
 281         0xa208, 0xffffffff, 0x100,
 282         0xa20c, 0xffffffff, 0x100,
 283         0x971c, 0xffffffff, 0x100,
 284         0x915c, 0xffffffff, 0x00020001,
 285         0x9160, 0xffffffff, 0x00040003,
 286         0x916c, 0xffffffff, 0x00060005,
 287         0x9170, 0xffffffff, 0x00080007,
 288         0x9174, 0xffffffff, 0x000a0009,
 289         0x9178, 0xffffffff, 0x000c000b,
 290         0x917c, 0xffffffff, 0x000e000d,
 291         0x9180, 0xffffffff, 0x0010000f,
 292         0x918c, 0xffffffff, 0x00120011,
 293         0x9190, 0xffffffff, 0x00140013,
 294         0x9194, 0xffffffff, 0x00020001,
 295         0x9198, 0xffffffff, 0x00040003,
 296         0x919c, 0xffffffff, 0x00060005,
 297         0x91a8, 0xffffffff, 0x00080007,
 298         0x91ac, 0xffffffff, 0x000a0009,
 299         0x91b0, 0xffffffff, 0x000c000b,
 300         0x91b4, 0xffffffff, 0x000e000d,
 301         0x91b8, 0xffffffff, 0x0010000f,
 302         0x91c4, 0xffffffff, 0x00120011,
 303         0x91c8, 0xffffffff, 0x00140013,
 304         0x91cc, 0xffffffff, 0x00020001,
 305         0x91d0, 0xffffffff, 0x00040003,
 306         0x91d4, 0xffffffff, 0x00060005,
 307         0x91e0, 0xffffffff, 0x00080007,
 308         0x91e4, 0xffffffff, 0x000a0009,
 309         0x91e8, 0xffffffff, 0x000c000b,
 310         0x91ec, 0xffffffff, 0x00020001,
 311         0x91f0, 0xffffffff, 0x00040003,
 312         0x91f4, 0xffffffff, 0x00060005,
 313         0x9200, 0xffffffff, 0x00080007,
 314         0x9204, 0xffffffff, 0x000a0009,
 315         0x9208, 0xffffffff, 0x000c000b,
 316         0x920c, 0xffffffff, 0x000e000d,
 317         0x9210, 0xffffffff, 0x0010000f,
 318         0x921c, 0xffffffff, 0x00120011,
 319         0x9220, 0xffffffff, 0x00140013,
 320         0x9224, 0xffffffff, 0x00020001,
 321         0x9228, 0xffffffff, 0x00040003,
 322         0x922c, 0xffffffff, 0x00060005,
 323         0x9238, 0xffffffff, 0x00080007,
 324         0x923c, 0xffffffff, 0x000a0009,
 325         0x9240, 0xffffffff, 0x000c000b,
 326         0x9244, 0xffffffff, 0x000e000d,
 327         0x9248, 0xffffffff, 0x0010000f,
 328         0x9254, 0xffffffff, 0x00120011,
 329         0x9258, 0xffffffff, 0x00140013,
 330         0x925c, 0xffffffff, 0x00020001,
 331         0x9260, 0xffffffff, 0x00040003,
 332         0x9264, 0xffffffff, 0x00060005,
 333         0x9270, 0xffffffff, 0x00080007,
 334         0x9274, 0xffffffff, 0x000a0009,
 335         0x9278, 0xffffffff, 0x000c000b,
 336         0x927c, 0xffffffff, 0x000e000d,
 337         0x9280, 0xffffffff, 0x0010000f,
 338         0x928c, 0xffffffff, 0x00120011,
 339         0x9290, 0xffffffff, 0x00140013,
 340         0x9294, 0xffffffff, 0x00020001,
 341         0x929c, 0xffffffff, 0x00040003,
 342         0x92a0, 0xffffffff, 0x00060005,
 343         0x92a4, 0xffffffff, 0x00080007
 344 };
 345
 346 static const u32 rv710_golden_registers[] =
 347 {
 348         0x3f90, 0x00ff0000, 0x00fc0000,
 349         0x9148, 0x00ff0000, 0x00fc0000,
 350         0x3f94, 0x00ff0000, 0x00fc0000,
 351         0x914c, 0x00ff0000, 0x00fc0000,
 352         0xb4c, 0x00000020, 0x00000020,
 353         0xa180, 0xffffffff, 0x00003f3f
 354 };
 355
 356 static const u32 rv710_mgcg_init[] =
 357 {
 358         0x8bcc, 0xffffffff, 0x13030040,
 359         0x5448, 0xffffffff, 0x100,
 360         0x55e4, 0xffffffff, 0x100,
 361         0x160c, 0xffffffff, 0x100,
 362         0x5644, 0xffffffff, 0x100,
 363         0xc164, 0xffffffff, 0x100,
 364         0x8a18, 0xffffffff, 0x100,
 365         0x897c, 0xffffffff, 0x8000100,
 366         0x8b28, 0xffffffff, 0x3c000100,
 367         0x9144, 0xffffffff, 0x100,
 368         0x9a1c, 0xffffffff, 0x10000,
 369         0x9a50, 0xffffffff, 0x100,
 370         0x9a1c, 0xffffffff, 0x0,
 371         0x9870, 0xffffffff, 0x100,
 372         0x8d58, 0xffffffff, 0x100,
 373         0x9500, 0xffffffff, 0x0,
 374         0x9510, 0xffffffff, 0x100,
 375         0x9500, 0xffffffff, 0x1,
 376         0x9510, 0xffffffff, 0x100,
 377         0x9500, 0xffffffff, 0x8000,
 378         0x9490, 0xffffffff, 0x0,
 379         0x949c, 0xffffffff, 0x100,
 380         0x9490, 0xffffffff, 0x1,
 381         0x949c, 0xffffffff, 0x100,
 382         0x9490, 0xffffffff, 0x8000,
 383         0x9604, 0xffffffff, 0x0,
 384         0x9654, 0xffffffff, 0x100,
 385         0x9604, 0xffffffff, 0x1,
 386         0x9654, 0xffffffff, 0x100,
 387         0x9604, 0xffffffff, 0x80000000,
 388         0x9030, 0xffffffff, 0x100,
 389         0x9034, 0xffffffff, 0x100,
 390         0x9038, 0xffffffff, 0x100,
 391         0x903c, 0xffffffff, 0x100,
 392         0x9040, 0xffffffff, 0x100,
 393         0xa200, 0xffffffff, 0x100,
 394         0xa204, 0xffffffff, 0x100,
 395         0xa208, 0xffffffff, 0x100,
 396         0xa20c, 0xffffffff, 0x100,
 397         0x971c, 0xffffffff, 0x100,
 398         0x915c, 0xffffffff, 0x00020001,
 399         0x9174, 0xffffffff, 0x00000003,
 400         0x9178, 0xffffffff, 0x00050001,
 401         0x917c, 0xffffffff, 0x00030002,
 402         0x918c, 0xffffffff, 0x00000004,
 403         0x9190, 0xffffffff, 0x00070006,
 404         0x9194, 0xffffffff, 0x00050001,
 405         0x9198, 0xffffffff, 0x00030002,
 406         0x91a8, 0xffffffff, 0x00000004,
 407         0x91ac, 0xffffffff, 0x00070006,
 408         0x91e8, 0xffffffff, 0x00000001,
 409         0x9294, 0xffffffff, 0x00000001,
 410         0x929c, 0xffffffff, 0x00000002,
 411         0x92a0, 0xffffffff, 0x00040003,
 412         0x9150, 0xffffffff, 0x4d940000
 413 };
 414
 415 static const u32 rv730_golden_registers[] =
 416 {
 417         0x3f90, 0x00ff0000, 0x00f00000,
 418         0x9148, 0x00ff0000, 0x00f00000,
 419         0x3f94, 0x00ff0000, 0x00f00000,
 420         0x914c, 0x00ff0000, 0x00f00000,
 421         0x900c, 0xffffffff, 0x003b033f,
 422         0xb4c, 0x00000020, 0x00000020,
 423         0xa180, 0xffffffff, 0x00003f3f
 424 };
 425
 426 static const u32 rv730_mgcg_init[] =
 427 {
 428         0x8bcc, 0xffffffff, 0x130300f9,
 429         0x5448, 0xffffffff, 0x100,
 430         0x55e4, 0xffffffff, 0x100,
 431         0x160c, 0xffffffff, 0x100,
 432         0x5644, 0xffffffff, 0x100,
 433         0xc164, 0xffffffff, 0x100,
 434         0x8a18, 0xffffffff, 0x100,
 435         0x897c, 0xffffffff, 0x8000100,
 436         0x8b28, 0xffffffff, 0x3c000100,
 437         0x9144, 0xffffffff, 0x100,
 438         0x9a1c, 0xffffffff, 0x10000,
 439         0x9a50, 0xffffffff, 0x100,
 440         0x9a1c, 0xffffffff, 0x10001,
 441         0x9a50, 0xffffffff, 0x100,
 442         0x9a1c, 0xffffffff, 0x0,
 443         0x9870, 0xffffffff, 0x100,
 444         0x8d58, 0xffffffff, 0x100,
 445         0x9500, 0xffffffff, 0x0,
 446         0x9510, 0xffffffff, 0x100,
 447         0x9500, 0xffffffff, 0x1,
 448         0x9510, 0xffffffff, 0x100,
 449         0x9500, 0xffffffff, 0x2,
 450         0x9510, 0xffffffff, 0x100,
 451         0x9500, 0xffffffff, 0x3,
 452         0x9510, 0xffffffff, 0x100,
 453         0x9500, 0xffffffff, 0x4,
 454         0x9510, 0xffffffff, 0x100,
 455         0x9500, 0xffffffff, 0x5,
 456         0x9510, 0xffffffff, 0x100,
 457         0x9500, 0xffffffff, 0x6,
 458         0x9510, 0xffffffff, 0x100,
 459         0x9500, 0xffffffff, 0x7,
 460         0x9510, 0xffffffff, 0x100,
 461         0x9500, 0xffffffff, 0x8000,
 462         0x9490, 0xffffffff, 0x0,
 463         0x949c, 0xffffffff, 0x100,
 464         0x9490, 0xffffffff, 0x1,
 465         0x949c, 0xffffffff, 0x100,
 466         0x9490, 0xffffffff, 0x2,
 467         0x949c, 0xffffffff, 0x100,
 468         0x9490, 0xffffffff, 0x3,
 469         0x949c, 0xffffffff, 0x100,
 470         0x9490, 0xffffffff, 0x4,
 471         0x949c, 0xffffffff, 0x100,
 472         0x9490, 0xffffffff, 0x5,
 473         0x949c, 0xffffffff, 0x100,
 474         0x9490, 0xffffffff, 0x6,
 475         0x949c, 0xffffffff, 0x100,
 476         0x9490, 0xffffffff, 0x7,
 477         0x949c, 0xffffffff, 0x100,
 478         0x9490, 0xffffffff, 0x8000,
 479         0x9604, 0xffffffff, 0x0,
 480         0x9654, 0xffffffff, 0x100,
 481         0x9604, 0xffffffff, 0x1,
 482         0x9654, 0xffffffff, 0x100,
 483         0x9604, 0xffffffff, 0x2,
 484         0x9654, 0xffffffff, 0x100,
 485         0x9604, 0xffffffff, 0x3,
 486         0x9654, 0xffffffff, 0x100,
 487         0x9604, 0xffffffff, 0x4,
 488         0x9654, 0xffffffff, 0x100,
 489         0x9604, 0xffffffff, 0x5,
 490         0x9654, 0xffffffff, 0x100,
 491         0x9604, 0xffffffff, 0x6,
 492         0x9654, 0xffffffff, 0x100,
 493         0x9604, 0xffffffff, 0x7,
 494         0x9654, 0xffffffff, 0x100,
 495         0x9604, 0xffffffff, 0x80000000,
 496         0x9030, 0xffffffff, 0x100,
 497         0x9034, 0xffffffff, 0x100,
 498         0x9038, 0xffffffff, 0x100,
 499         0x903c, 0xffffffff, 0x100,
 500         0x9040, 0xffffffff, 0x100,
 501         0xa200, 0xffffffff, 0x100,
 502         0xa204, 0xffffffff, 0x100,
 503         0xa208, 0xffffffff, 0x100,
 504         0xa20c, 0xffffffff, 0x100,
 505         0x971c, 0xffffffff, 0x100,
 506         0x915c, 0xffffffff, 0x00020001,
 507         0x916c, 0xffffffff, 0x00040003,
 508         0x9170, 0xffffffff, 0x00000005,
 509         0x9178, 0xffffffff, 0x00050001,
 510         0x917c, 0xffffffff, 0x00030002,
 511         0x918c, 0xffffffff, 0x00000004,
 512         0x9190, 0xffffffff, 0x00070006,
 513         0x9194, 0xffffffff, 0x00050001,
 514         0x9198, 0xffffffff, 0x00030002,
 515         0x91a8, 0xffffffff, 0x00000004,
 516         0x91ac, 0xffffffff, 0x00070006,
 517         0x91b0, 0xffffffff, 0x00050001,
 518         0x91b4, 0xffffffff, 0x00030002,
 519         0x91c4, 0xffffffff, 0x00000004,
 520         0x91c8, 0xffffffff, 0x00070006,
 521         0x91cc, 0xffffffff, 0x00050001,
 522         0x91d0, 0xffffffff, 0x00030002,
 523         0x91e0, 0xffffffff, 0x00000004,
 524         0x91e4, 0xffffffff, 0x00070006,
 525         0x91e8, 0xffffffff, 0x00000001,
 526         0x91ec, 0xffffffff, 0x00050001,
 527         0x91f0, 0xffffffff, 0x00030002,
 528         0x9200, 0xffffffff, 0x00000004,
 529         0x9204, 0xffffffff, 0x00070006,
 530         0x9208, 0xffffffff, 0x00050001,
 531         0x920c, 0xffffffff, 0x00030002,
 532         0x921c, 0xffffffff, 0x00000004,
 533         0x9220, 0xffffffff, 0x00070006,
 534         0x9224, 0xffffffff, 0x00050001,
 535         0x9228, 0xffffffff, 0x00030002,
 536         0x9238, 0xffffffff, 0x00000004,
 537         0x923c, 0xffffffff, 0x00070006,
 538         0x9240, 0xffffffff, 0x00050001,
 539         0x9244, 0xffffffff, 0x00030002,
 540         0x9254, 0xffffffff, 0x00000004,
 541         0x9258, 0xffffffff, 0x00070006,
 542         0x9294, 0xffffffff, 0x00000001,
 543         0x929c, 0xffffffff, 0x00000002,
 544         0x92a0, 0xffffffff, 0x00040003,
 545         0x92a4, 0xffffffff, 0x00000005
 546 };
 547
 548 static const u32 rv740_golden_registers[] =
 549 {
 550         0x88c4, 0xffffffff, 0x00000082,
 551         0x28a50, 0xfffffffc, 0x00000004,
 552         0x2650, 0x00040000, 0,
 553         0x20bc, 0x00040000, 0,
 554         0x733c, 0xffffffff, 0x00000002,
 555         0x7300, 0xffffffff, 0x001000f0,
 556         0x3f90, 0x00ff0000, 0,
 557         0x9148, 0x00ff0000, 0,
 558         0x3f94, 0x00ff0000, 0,
 559         0x914c, 0x00ff0000, 0,
 560         0x240c, 0xffffffff, 0x00000380,
 561         0x8a14, 0x00000007, 0x00000007,
 562         0x8b24, 0xffffffff, 0x00ff0fff,
 563         0x28a4c, 0xffffffff, 0x00004000,
 564         0xa180, 0xffffffff, 0x00003f3f,
 565         0x8d00, 0xffffffff, 0x0e0e003a,
 566         0x8d04, 0xffffffff, 0x013a0e2a,
 567         0x8c00, 0xffffffff, 0xe400000f,
 568         0x8db0, 0xffffffff, 0x98989898,
 569         0x8db4, 0xffffffff, 0x98989898,
 570         0x8db8, 0xffffffff, 0x98989898,
 571         0x8dbc, 0xffffffff, 0x98989898,
 572         0x8dc0, 0xffffffff, 0x98989898,
 573         0x8dc4, 0xffffffff, 0x98989898,
 574         0x8dc8, 0xffffffff, 0x98989898,
 575         0x8dcc, 0xffffffff, 0x98989898,
 576         0x9058, 0xffffffff, 0x0fffc40f,
 577         0x900c, 0xffffffff, 0x003b033f,
 578         0x28350, 0xffffffff, 0,
 579         0x8cf0, 0x1fffffff, 0x08e00420,
 580         0x9508, 0xffffffff, 0x00000002,
 581         0x88c4, 0xffffffff, 0x000000c2,
 582         0x9698, 0x18000000, 0x18000000
 583 };
 584
 585 static const u32 rv740_mgcg_init[] =
 586 {
 587         0x8bcc, 0xffffffff, 0x13030100,
 588         0x5448, 0xffffffff, 0x100,
 589         0x55e4, 0xffffffff, 0x100,
 590         0x160c, 0xffffffff, 0x100,
 591         0x5644, 0xffffffff, 0x100,
 592         0xc164, 0xffffffff, 0x100,
 593         0x8a18, 0xffffffff, 0x100,
 594         0x897c, 0xffffffff, 0x100,
 595         0x8b28, 0xffffffff, 0x100,
 596         0x9144, 0xffffffff, 0x100,
 597         0x9a1c, 0xffffffff, 0x10000,
 598         0x9a50, 0xffffffff, 0x100,
 599         0x9a1c, 0xffffffff, 0x10001,
 600         0x9a50, 0xffffffff, 0x100,
 601         0x9a1c, 0xffffffff, 0x10002,
 602         0x9a50, 0xffffffff, 0x100,
 603         0x9a1c, 0xffffffff, 0x10003,
 604         0x9a50, 0xffffffff, 0x100,
 605         0x9a1c, 0xffffffff, 0x0,
 606         0x9870, 0xffffffff, 0x100,
 607         0x8d58, 0xffffffff, 0x100,
 608         0x9500, 0xffffffff, 0x0,
 609         0x9510, 0xffffffff, 0x100,
 610         0x9500, 0xffffffff, 0x1,
 611         0x9510, 0xffffffff, 0x100,
 612         0x9500, 0xffffffff, 0x2,
 613         0x9510, 0xffffffff, 0x100,
 614         0x9500, 0xffffffff, 0x3,
 615         0x9510, 0xffffffff, 0x100,
 616         0x9500, 0xffffffff, 0x4,
 617         0x9510, 0xffffffff, 0x100,
 618         0x9500, 0xffffffff, 0x5,
 619         0x9510, 0xffffffff, 0x100,
 620         0x9500, 0xffffffff, 0x6,
 621         0x9510, 0xffffffff, 0x100,
 622         0x9500, 0xffffffff, 0x7,
 623         0x9510, 0xffffffff, 0x100,
 624         0x9500, 0xffffffff, 0x8000,
 625         0x9490, 0xffffffff, 0x0,
 626         0x949c, 0xffffffff, 0x100,
 627         0x9490, 0xffffffff, 0x1,
 628         0x949c, 0xffffffff, 0x100,
 629         0x9490, 0xffffffff, 0x2,
 630         0x949c, 0xffffffff, 0x100,
 631         0x9490, 0xffffffff, 0x3,
 632         0x949c, 0xffffffff, 0x100,
 633         0x9490, 0xffffffff, 0x4,
 634         0x949c, 0xffffffff, 0x100,
 635         0x9490, 0xffffffff, 0x5,
 636         0x949c, 0xffffffff, 0x100,
 637         0x9490, 0xffffffff, 0x6,
 638         0x949c, 0xffffffff, 0x100,
 639         0x9490, 0xffffffff, 0x7,
 640         0x949c, 0xffffffff, 0x100,
 641         0x9490, 0xffffffff, 0x8000,
 642         0x9604, 0xffffffff, 0x0,
 643         0x9654, 0xffffffff, 0x100,
 644         0x9604, 0xffffffff, 0x1,
 645         0x9654, 0xffffffff, 0x100,
 646         0x9604, 0xffffffff, 0x2,
 647         0x9654, 0xffffffff, 0x100,
 648         0x9604, 0xffffffff, 0x3,
 649         0x9654, 0xffffffff, 0x100,
 650         0x9604, 0xffffffff, 0x4,
 651         0x9654, 0xffffffff, 0x100,
 652         0x9604, 0xffffffff, 0x5,
 653         0x9654, 0xffffffff, 0x100,
 654         0x9604, 0xffffffff, 0x6,
 655         0x9654, 0xffffffff, 0x100,
 656         0x9604, 0xffffffff, 0x7,
 657         0x9654, 0xffffffff, 0x100,
 658         0x9604, 0xffffffff, 0x80000000,
 659         0x9030, 0xffffffff, 0x100,
 660         0x9034, 0xffffffff, 0x100,
 661         0x9038, 0xffffffff, 0x100,
 662         0x903c, 0xffffffff, 0x100,
 663         0x9040, 0xffffffff, 0x100,
 664         0xa200, 0xffffffff, 0x100,
 665         0xa204, 0xffffffff, 0x100,
 666         0xa208, 0xffffffff, 0x100,
 667         0xa20c, 0xffffffff, 0x100,
 668         0x971c, 0xffffffff, 0x100,
 669         0x915c, 0xffffffff, 0x00020001,
 670         0x9160, 0xffffffff, 0x00040003,
 671         0x916c, 0xffffffff, 0x00060005,
 672         0x9170, 0xffffffff, 0x00080007,
 673         0x9174, 0xffffffff, 0x000a0009,
 674         0x9178, 0xffffffff, 0x000c000b,
 675         0x917c, 0xffffffff, 0x000e000d,
 676         0x9180, 0xffffffff, 0x0010000f,
 677         0x918c, 0xffffffff, 0x00120011,
 678         0x9190, 0xffffffff, 0x00140013,
 679         0x9194, 0xffffffff, 0x00020001,
 680         0x9198, 0xffffffff, 0x00040003,
 681         0x919c, 0xffffffff, 0x00060005,
 682         0x91a8, 0xffffffff, 0x00080007,
 683         0x91ac, 0xffffffff, 0x000a0009,
 684         0x91b0, 0xffffffff, 0x000c000b,
 685         0x91b4, 0xffffffff, 0x000e000d,
 686         0x91b8, 0xffffffff, 0x0010000f,
 687         0x91c4, 0xffffffff, 0x00120011,
 688         0x91c8, 0xffffffff, 0x00140013,
 689         0x91cc, 0xffffffff, 0x00020001,
 690         0x91d0, 0xffffffff, 0x00040003,
 691         0x91d4, 0xffffffff, 0x00060005,
 692         0x91e0, 0xffffffff, 0x00080007,
 693         0x91e4, 0xffffffff, 0x000a0009,
 694         0x91e8, 0xffffffff, 0x000c000b,
 695         0x91ec, 0xffffffff, 0x00020001,
 696         0x91f0, 0xffffffff, 0x00040003,
 697         0x91f4, 0xffffffff, 0x00060005,
 698         0x9200, 0xffffffff, 0x00080007,
 699         0x9204, 0xffffffff, 0x000a0009,
 700         0x9208, 0xffffffff, 0x000c000b,
 701         0x920c, 0xffffffff, 0x000e000d,
 702         0x9210, 0xffffffff, 0x0010000f,
 703         0x921c, 0xffffffff, 0x00120011,
 704         0x9220, 0xffffffff, 0x00140013,
 705         0x9224, 0xffffffff, 0x00020001,
 706         0x9228, 0xffffffff, 0x00040003,
 707         0x922c, 0xffffffff, 0x00060005,
 708         0x9238, 0xffffffff, 0x00080007,
 709         0x923c, 0xffffffff, 0x000a0009,
 710         0x9240, 0xffffffff, 0x000c000b,
 711         0x9244, 0xffffffff, 0x000e000d,
 712         0x9248, 0xffffffff, 0x0010000f,
 713         0x9254, 0xffffffff, 0x00120011,
 714         0x9258, 0xffffffff, 0x00140013,
 715         0x9294, 0xffffffff, 0x00020001,
 716         0x929c, 0xffffffff, 0x00040003,
 717         0x92a0, 0xffffffff, 0x00060005,
 718         0x92a4, 0xffffffff, 0x00080007
 719 };
 720
 721 static void rv770_init_golden_registers(struct radeon_device *rdev)
 722 {
 723         switch (rdev->family) {
 724         case CHIP_RV770:
 725                 radeon_program_register_sequence(rdev,
 726                                                  r7xx_golden_registers,
 727                                                  (const u32)ARRAY_SIZE(r7xx_golden_registers));
 728                 radeon_program_register_sequence(rdev,
 729                                                  r7xx_golden_dyn_gpr_registers,
 730                                                  (const u32)ARRAY_SIZE(r7xx_golden_dyn_gpr_registers));
 731                 if (rdev->pdev->device == 0x994e)
 732                         radeon_program_register_sequence(rdev,
 733                                                          rv770ce_golden_registers,
 734                                                          (const u32)ARRAY_SIZE(rv770ce_golden_registers));
 735                 else
 736                         radeon_program_register_sequence(rdev,
 737                                                          rv770_golden_registers,
 738                                                          (const u32)ARRAY_SIZE(rv770_golden_registers));
 739                 radeon_program_register_sequence(rdev,
 740                                                  rv770_mgcg_init,
 741                                                  (const u32)ARRAY_SIZE(rv770_mgcg_init));
 742                 break;
 743         case CHIP_RV730:
 744                 radeon_program_register_sequence(rdev,
 745                                                  r7xx_golden_registers,
 746                                                  (const u32)ARRAY_SIZE(r7xx_golden_registers));
 747                 radeon_program_register_sequence(rdev,
 748                                                  r7xx_golden_dyn_gpr_registers,
 749                                                  (const u32)ARRAY_SIZE(r7xx_golden_dyn_gpr_registers));
 750                 radeon_program_register_sequence(rdev,
 751                                                  rv730_golden_registers,
 752                                                  (const u32)ARRAY_SIZE(rv730_golden_registers));
 753                 radeon_program_register_sequence(rdev,
 754                                                  rv730_mgcg_init,
 755                                                  (const u32)ARRAY_SIZE(rv730_mgcg_init));
 756                 break;
 757         case CHIP_RV710:
 758                 radeon_program_register_sequence(rdev,
 759                                                  r7xx_golden_registers,
 760                                                  (const u32)ARRAY_SIZE(r7xx_golden_registers));
 761                 radeon_program_register_sequence(rdev,
 762                                                  r7xx_golden_dyn_gpr_registers,
 763                                                  (const u32)ARRAY_SIZE(r7xx_golden_dyn_gpr_registers));
 764                 radeon_program_register_sequence(rdev,
 765                                                  rv710_golden_registers,
 766                                                  (const u32)ARRAY_SIZE(rv710_golden_registers));
 767                 radeon_program_register_sequence(rdev,
 768                                                  rv710_mgcg_init,
 769                                                  (const u32)ARRAY_SIZE(rv710_mgcg_init));
 770                 break;
 771         case CHIP_RV740:
 772                 radeon_program_register_sequence(rdev,
 773                                                  rv740_golden_registers,
 774                                                  (const u32)ARRAY_SIZE(rv740_golden_registers));
 775                 radeon_program_register_sequence(rdev,
 776                                                  rv740_mgcg_init,
 777                                                  (const u32)ARRAY_SIZE(rv740_mgcg_init));
 778                 break;
 779         default:
 780                 break;
 781         }
 782 }
 783
 784 #define PCIE_BUS_CLK                10000
 785 #define TCLK                        (PCIE_BUS_CLK / 10)
 786
 787 /**
 788  * rv770_get_xclk - get the xclk
 789  *
 790  * @rdev: radeon_device pointer
 791  *
 792  * Returns the reference clock used by the gfx engine
 793  * (r7xx-cayman).
 794  */
 795 u32 rv770_get_xclk(struct radeon_device *rdev)
 796 {
 797         u32 reference_clock = rdev->clock.spll.reference_freq;
 798         u32 tmp = RREG32(CG_CLKPIN_CNTL);
 799
 800         if (tmp & MUX_TCLK_TO_XCLK)
 801                 return TCLK;
 802
 803         if (tmp & XTALIN_DIVIDE)
 804                 return reference_clock / 4;
 805
 806         return reference_clock;
 807 }
 808
 809 void rv770_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base, bool async)
 810 {
 811         struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
 812         u32 tmp = RREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset);
 813         int i;
 814
 815         /* Lock the graphics update lock */
 816         tmp |= AVIVO_D1GRPH_UPDATE_LOCK;
 817         WREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
 818
 819         /* update the scanout addresses */
 820         WREG32(AVIVO_D1GRPH_FLIP_CONTROL + radeon_crtc->crtc_offset,
 821                async ? AVIVO_D1GRPH_SURFACE_UPDATE_H_RETRACE_EN : 0);
 822         if (radeon_crtc->crtc_id) {
 823                 WREG32(D2GRPH_SECONDARY_SURFACE_ADDRESS_HIGH, upper_32_bits(crtc_base));
 824                 WREG32(D2GRPH_PRIMARY_SURFACE_ADDRESS_HIGH, upper_32_bits(crtc_base));
 825         } else {
 826                 WREG32(D1GRPH_SECONDARY_SURFACE_ADDRESS_HIGH, upper_32_bits(crtc_base));
 827                 WREG32(D1GRPH_PRIMARY_SURFACE_ADDRESS_HIGH, upper_32_bits(crtc_base));
 828         }
 829         WREG32(D1GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
 830                (u32)crtc_base);
 831         WREG32(D1GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
 832                (u32)crtc_base);
 833
 834         /* Wait for update_pending to go high. */
 835         for (i = 0; i < rdev->usec_timeout; i++) {
 836                 if (RREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset) & AVIVO_D1GRPH_SURFACE_UPDATE_PENDING)
 837                         break;
 838                 udelay(1);
 839         }
 840         DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
 841
 842         /* Unlock the lock, so double-buffering can take place inside vblank */
 843         tmp &= ~AVIVO_D1GRPH_UPDATE_LOCK;
 844         WREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
 845 }
 846
 847 bool rv770_page_flip_pending(struct radeon_device *rdev, int crtc_id)
 848 {
 849         struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
 850
 851         /* Return current update_pending status: */
 852         return !!(RREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset) &
 853                 AVIVO_D1GRPH_SURFACE_UPDATE_PENDING);
 854 }
 855
 856 /* get temperature in millidegrees */
 857 int rv770_get_temp(struct radeon_device *rdev)
 858 {
 859         u32 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
 860                 ASIC_T_SHIFT;
 861         int actual_temp;
 862
 863         if (temp & 0x400)
 864                 actual_temp = -256;
 865         else if (temp & 0x200)
 866                 actual_temp = 255;
 867         else if (temp & 0x100) {
 868                 actual_temp = temp & 0x1ff;
 869                 actual_temp |= ~0x1ff;
 870         } else
 871                 actual_temp = temp & 0xff;
 872
 873         return (actual_temp * 1000) / 2;
 874 }
 875
 876 void rv770_pm_misc(struct radeon_device *rdev)
 877 {
 878         int req_ps_idx = rdev->pm.requested_power_state_index;
 879         int req_cm_idx = rdev->pm.requested_clock_mode_index;
 880         struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
 881         struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
 882
 883         if ((voltage->type == VOLTAGE_SW) && voltage->voltage) {
 884                 /* 0xff01 is a flag rather then an actual voltage */
 885                 if (voltage->voltage == 0xff01)
 886                         return;
 887                 if (voltage->voltage != rdev->pm.current_vddc) {
 888                         radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
 889                         rdev->pm.current_vddc = voltage->voltage;
 890                         DRM_DEBUG("Setting: v: %d\n", voltage->voltage);
 891                 }
 892         }
 893 }
 894
 895 /*
 896  * GART
 897  */
 898 static int rv770_pcie_gart_enable(struct radeon_device *rdev)
 899 {
 900         u32 tmp;
 901         int r, i;
 902
 903         if (rdev->gart.robj == NULL) {
 904                 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
 905                 return -EINVAL;
 906         }
 907         r = radeon_gart_table_vram_pin(rdev);
 908         if (r)
 909                 return r;
 910         /* Setup L2 cache */
 911         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
 912                                 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
 913                                 EFFECTIVE_L2_QUEUE_SIZE(7));
 914         WREG32(VM_L2_CNTL2, 0);
 915         WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
 916         /* Setup TLB control */
 917         tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
 918                 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
 919                 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
 920                 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
 921         WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
 922         WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
 923         WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
 924         if (rdev->family == CHIP_RV740)
 925                 WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
 926         WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
 927         WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
 928         WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
 929         WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
 930         WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
 931         WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
 932         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
 933         WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
 934                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
 935         WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
 936                         (u32)(rdev->dummy_page.addr >> 12));
 937         for (i = 1; i < 7; i++)
 938                 WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
 939
 940         r600_pcie_gart_tlb_flush(rdev);
 941         DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
 942                  (unsigned)(rdev->mc.gtt_size >> 20),
 943                  (unsigned long long)rdev->gart.table_addr);
 944         rdev->gart.ready = true;
 945         return 0;
 946 }
 947
 948 static void rv770_pcie_gart_disable(struct radeon_device *rdev)
 949 {
 950         u32 tmp;
 951         int i;
 952
 953         /* Disable all tables */
 954         for (i = 0; i < 7; i++)
 955                 WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
 956
 957         /* Setup L2 cache */
 958         WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
 959                                 EFFECTIVE_L2_QUEUE_SIZE(7));
 960         WREG32(VM_L2_CNTL2, 0);
 961         WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
 962         /* Setup TLB control */
 963         tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
 964         WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
 965         WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
 966         WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
 967         WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
 968         WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
 969         WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
 970         WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
 971         radeon_gart_table_vram_unpin(rdev);
 972 }
 973
 974 static void rv770_pcie_gart_fini(struct radeon_device *rdev)
 975 {
 976         radeon_gart_fini(rdev);
 977         rv770_pcie_gart_disable(rdev);
 978         radeon_gart_table_vram_free(rdev);
 979 }
 980
 981
 982 static void rv770_agp_enable(struct radeon_device *rdev)
 983 {
 984         u32 tmp;
 985         int i;
 986
 987         /* Setup L2 cache */
 988         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
 989                                 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
 990                                 EFFECTIVE_L2_QUEUE_SIZE(7));
 991         WREG32(VM_L2_CNTL2, 0);
 992         WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
 993         /* Setup TLB control */
 994         tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
 995                 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
 996                 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
 997                 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
 998         WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
 999         WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
1000         WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
1001         WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
1002         WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
1003         WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
1004         WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
1005         for (i = 0; i < 7; i++)
1006                 WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
1007 }
1008
1009 static void rv770_mc_program(struct radeon_device *rdev)
1010 {
1011         struct rv515_mc_save save;
1012         u32 tmp;
1013         int i, j;
1014
1015         /* Initialize HDP */
1016         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
1017                 WREG32((0x2c14 + j), 0x00000000);
1018                 WREG32((0x2c18 + j), 0x00000000);
1019                 WREG32((0x2c1c + j), 0x00000000);
1020                 WREG32((0x2c20 + j), 0x00000000);
1021                 WREG32((0x2c24 + j), 0x00000000);
1022         }
1023         /* r7xx hw bug.  Read from HDP_DEBUG1 rather
1024          * than writing to HDP_REG_COHERENCY_FLUSH_CNTL
1025          */
1026         tmp = RREG32(HDP_DEBUG1);
1027
1028         rv515_mc_stop(rdev, &save);
1029         if (r600_mc_wait_for_idle(rdev)) {
1030                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1031         }
1032         /* Lockout access through VGA aperture*/
1033         WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
1034         /* Update configuration */
1035         if (rdev->flags & RADEON_IS_AGP) {
1036                 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
1037                         /* VRAM before AGP */
1038                         WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
1039                                 rdev->mc.vram_start >> 12);
1040                         WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1041                                 rdev->mc.gtt_end >> 12);
1042                 } else {
1043                         /* VRAM after AGP */
1044                         WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
1045                                 rdev->mc.gtt_start >> 12);
1046                         WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1047                                 rdev->mc.vram_end >> 12);
1048                 }
1049         } else {
1050                 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
1051                         rdev->mc.vram_start >> 12);
1052                 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1053                         rdev->mc.vram_end >> 12);
1054         }
1055         WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
1056         tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
1057         tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
1058         WREG32(MC_VM_FB_LOCATION, tmp);
1059         WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
1060         WREG32(HDP_NONSURFACE_INFO, (2 << 7));
1061         WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
1062         if (rdev->flags & RADEON_IS_AGP) {
1063                 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
1064                 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
1065                 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
1066         } else {
1067                 WREG32(MC_VM_AGP_BASE, 0);
1068                 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
1069                 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
1070         }
1071         if (r600_mc_wait_for_idle(rdev)) {
1072                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1073         }
1074         rv515_mc_resume(rdev, &save);
1075         /* we need to own VRAM, so turn off the VGA renderer here
1076          * to stop it overwriting our objects */
1077         rv515_vga_render_disable(rdev);
1078 }
1079
1080
1081 /*
1082  * CP.
1083  */
1084 void r700_cp_stop(struct radeon_device *rdev)
1085 {
1086         if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
1087                 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
1088         WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT));
1089         WREG32(SCRATCH_UMSK, 0);
1090         rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
1091 }
1092
1093 static int rv770_cp_load_microcode(struct radeon_device *rdev)
1094 {
1095         const __be32 *fw_data;
1096         int i;
1097
1098         if (!rdev->me_fw || !rdev->pfp_fw)
1099                 return -EINVAL;
1100
1101         r700_cp_stop(rdev);
1102         WREG32(CP_RB_CNTL,
1103 #ifdef __BIG_ENDIAN
1104                BUF_SWAP_32BIT |
1105 #endif
1106                RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
1107
1108         /* Reset cp */
1109         WREG32(GRBM_SOFT_RESET, SOFT_RESET_CP);
1110         RREG32(GRBM_SOFT_RESET);
1111         mdelay(15);
1112         WREG32(GRBM_SOFT_RESET, 0);
1113
1114         fw_data = (const __be32 *)rdev->pfp_fw->data;
1115         WREG32(CP_PFP_UCODE_ADDR, 0);
1116         for (i = 0; i < R700_PFP_UCODE_SIZE; i++)
1117                 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
1118         WREG32(CP_PFP_UCODE_ADDR, 0);
1119
1120         fw_data = (const __be32 *)rdev->me_fw->data;
1121         WREG32(CP_ME_RAM_WADDR, 0);
1122         for (i = 0; i < R700_PM4_UCODE_SIZE; i++)
1123                 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
1124
1125         WREG32(CP_PFP_UCODE_ADDR, 0);
1126         WREG32(CP_ME_RAM_WADDR, 0);
1127         WREG32(CP_ME_RAM_RADDR, 0);
1128         return 0;
1129 }
1130
1131 void r700_cp_fini(struct radeon_device *rdev)
1132 {
1133         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1134         r700_cp_stop(rdev);
1135         radeon_ring_fini(rdev, ring);
1136         radeon_scratch_free(rdev, ring->rptr_save_reg);
1137 }
1138
1139 void rv770_set_clk_bypass_mode(struct radeon_device *rdev)
1140 {
1141         u32 tmp, i;
1142
1143         if (rdev->flags & RADEON_IS_IGP)
1144                 return;
1145
1146         tmp = RREG32(CG_SPLL_FUNC_CNTL_2);
1147         tmp &= SCLK_MUX_SEL_MASK;
1148         tmp |= SCLK_MUX_SEL(1) | SCLK_MUX_UPDATE;
1149         WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
1150
1151         for (i = 0; i < rdev->usec_timeout; i++) {
1152                 if (RREG32(CG_SPLL_STATUS) & SPLL_CHG_STATUS)
1153                         break;
1154                 udelay(1);
1155         }
1156
1157         tmp &= ~SCLK_MUX_UPDATE;
1158         WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
1159
1160         tmp = RREG32(MPLL_CNTL_MODE);
1161         if ((rdev->family == CHIP_RV710) || (rdev->family == CHIP_RV730))
1162                 tmp &= ~RV730_MPLL_MCLK_SEL;
1163         else
1164                 tmp &= ~MPLL_MCLK_SEL;
1165         WREG32(MPLL_CNTL_MODE, tmp);
1166 }
1167
1168 /*
1169  * Core functions
1170  */
1171 static void rv770_gpu_init(struct radeon_device *rdev)
1172 {
1173         int i, j, num_qd_pipes;
1174         u32 ta_aux_cntl;
1175         u32 sx_debug_1;
1176         u32 smx_dc_ctl0;
1177         u32 db_debug3;
1178         u32 num_gs_verts_per_thread;
1179         u32 vgt_gs_per_es;
1180         u32 gs_prim_buffer_depth = 0;
1181         u32 sq_ms_fifo_sizes;
1182         u32 sq_config;
1183         u32 sq_thread_resource_mgmt;
1184         u32 hdp_host_path_cntl;
1185         u32 sq_dyn_gpr_size_simd_ab_0;
1186         u32 gb_tiling_config = 0;
1187         u32 cc_gc_shader_pipe_config = 0;
1188         u32 mc_arb_ramcfg;
1189         u32 db_debug4, tmp;
1190         u32 inactive_pipes, shader_pipe_config;
1191         u32 disabled_rb_mask;
1192         unsigned active_number;
1193
1194         /* setup chip specs */
1195         rdev->config.rv770.tiling_group_size = 256;
1196         switch (rdev->family) {
1197         case CHIP_RV770:
1198                 rdev->config.rv770.max_pipes = 4;
1199                 rdev->config.rv770.max_tile_pipes = 8;
1200                 rdev->config.rv770.max_simds = 10;
1201                 rdev->config.rv770.max_backends = 4;
1202                 rdev->config.rv770.max_gprs = 256;
1203                 rdev->config.rv770.max_threads = 248;
1204                 rdev->config.rv770.max_stack_entries = 512;
1205                 rdev->config.rv770.max_hw_contexts = 8;
1206                 rdev->config.rv770.max_gs_threads = 16 * 2;
1207                 rdev->config.rv770.sx_max_export_size = 128;
1208                 rdev->config.rv770.sx_max_export_pos_size = 16;
1209                 rdev->config.rv770.sx_max_export_smx_size = 112;
1210                 rdev->config.rv770.sq_num_cf_insts = 2;
1211
1212                 rdev->config.rv770.sx_num_of_sets = 7;
1213                 rdev->config.rv770.sc_prim_fifo_size = 0xF9;
1214                 rdev->config.rv770.sc_hiz_tile_fifo_size = 0x30;
1215                 rdev->config.rv770.sc_earlyz_tile_fifo_fize = 0x130;
1216                 break;
1217         case CHIP_RV730:
1218                 rdev->config.rv770.max_pipes = 2;
1219                 rdev->config.rv770.max_tile_pipes = 4;
1220                 rdev->config.rv770.max_simds = 8;
1221                 rdev->config.rv770.max_backends = 2;
1222                 rdev->config.rv770.max_gprs = 128;
1223                 rdev->config.rv770.max_threads = 248;
1224                 rdev->config.rv770.max_stack_entries = 256;
1225                 rdev->config.rv770.max_hw_contexts = 8;
1226                 rdev->config.rv770.max_gs_threads = 16 * 2;
1227                 rdev->config.rv770.sx_max_export_size = 256;
1228                 rdev->config.rv770.sx_max_export_pos_size = 32;
1229                 rdev->config.rv770.sx_max_export_smx_size = 224;
1230                 rdev->config.rv770.sq_num_cf_insts = 2;
1231
1232                 rdev->config.rv770.sx_num_of_sets = 7;
1233                 rdev->config.rv770.sc_prim_fifo_size = 0xf9;
1234                 rdev->config.rv770.sc_hiz_tile_fifo_size = 0x30;
1235                 rdev->config.rv770.sc_earlyz_tile_fifo_fize = 0x130;
1236                 if (rdev->config.rv770.sx_max_export_pos_size > 16) {
1237                         rdev->config.rv770.sx_max_export_pos_size -= 16;
1238                         rdev->config.rv770.sx_max_export_smx_size += 16;
1239                 }
1240                 break;
1241         case CHIP_RV710:
1242                 rdev->config.rv770.max_pipes = 2;
1243                 rdev->config.rv770.max_tile_pipes = 2;
1244                 rdev->config.rv770.max_simds = 2;
1245                 rdev->config.rv770.max_backends = 1;
1246                 rdev->config.rv770.max_gprs = 256;
1247                 rdev->config.rv770.max_threads = 192;
1248                 rdev->config.rv770.max_stack_entries = 256;
1249                 rdev->config.rv770.max_hw_contexts = 4;
1250                 rdev->config.rv770.max_gs_threads = 8 * 2;
1251                 rdev->config.rv770.sx_max_export_size = 128;
1252                 rdev->config.rv770.sx_max_export_pos_size = 16;
1253                 rdev->config.rv770.sx_max_export_smx_size = 112;
1254                 rdev->config.rv770.sq_num_cf_insts = 1;
1255
1256                 rdev->config.rv770.sx_num_of_sets = 7;
1257                 rdev->config.rv770.sc_prim_fifo_size = 0x40;
1258                 rdev->config.rv770.sc_hiz_tile_fifo_size = 0x30;
1259                 rdev->config.rv770.sc_earlyz_tile_fifo_fize = 0x130;
1260                 break;
1261         case CHIP_RV740:
1262                 rdev->config.rv770.max_pipes = 4;
1263                 rdev->config.rv770.max_tile_pipes = 4;
1264                 rdev->config.rv770.max_simds = 8;
1265                 rdev->config.rv770.max_backends = 4;
1266                 rdev->config.rv770.max_gprs = 256;
1267                 rdev->config.rv770.max_threads = 248;
1268                 rdev->config.rv770.max_stack_entries = 512;
1269                 rdev->config.rv770.max_hw_contexts = 8;
1270                 rdev->config.rv770.max_gs_threads = 16 * 2;
1271                 rdev->config.rv770.sx_max_export_size = 256;
1272                 rdev->config.rv770.sx_max_export_pos_size = 32;
1273                 rdev->config.rv770.sx_max_export_smx_size = 224;
1274                 rdev->config.rv770.sq_num_cf_insts = 2;
1275
1276                 rdev->config.rv770.sx_num_of_sets = 7;
1277                 rdev->config.rv770.sc_prim_fifo_size = 0x100;
1278                 rdev->config.rv770.sc_hiz_tile_fifo_size = 0x30;
1279                 rdev->config.rv770.sc_earlyz_tile_fifo_fize = 0x130;
1280
1281                 if (rdev->config.rv770.sx_max_export_pos_size > 16) {
1282                         rdev->config.rv770.sx_max_export_pos_size -= 16;
1283                         rdev->config.rv770.sx_max_export_smx_size += 16;
1284                 }
1285                 break;
1286         default:
1287                 break;
1288         }
1289
1290         /* Initialize HDP */
1291         j = 0;
1292         for (i = 0; i < 32; i++) {
1293                 WREG32((0x2c14 + j), 0x00000000);
1294                 WREG32((0x2c18 + j), 0x00000000);
1295                 WREG32((0x2c1c + j), 0x00000000);
1296                 WREG32((0x2c20 + j), 0x00000000);
1297                 WREG32((0x2c24 + j), 0x00000000);
1298                 j += 0x18;
1299         }
1300
1301         WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
1302
1303         /* setup tiling, simd, pipe config */
1304         mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
1305
1306         shader_pipe_config = RREG32(CC_GC_SHADER_PIPE_CONFIG);
1307         inactive_pipes = (shader_pipe_config & INACTIVE_QD_PIPES_MASK) >> INACTIVE_QD_PIPES_SHIFT;
1308         for (i = 0, tmp = 1, active_number = 0; i < R7XX_MAX_PIPES; i++) {
1309                 if (!(inactive_pipes & tmp)) {
1310                         active_number++;
1311                 }
1312                 tmp <<= 1;
1313         }
1314         if (active_number == 1) {
1315                 WREG32(SPI_CONFIG_CNTL, DISABLE_INTERP_1);
1316         } else {
1317                 WREG32(SPI_CONFIG_CNTL, 0);
1318         }
1319
1320         cc_gc_shader_pipe_config = RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffffff00;
1321         tmp = rdev->config.rv770.max_simds -
1322                 r600_count_pipe_bits((cc_gc_shader_pipe_config >> 16) & R7XX_MAX_SIMDS_MASK);
1323         rdev->config.rv770.active_simds = tmp;
1324
1325         switch (rdev->config.rv770.max_tile_pipes) {
1326         case 1:
1327         default:
1328                 gb_tiling_config = PIPE_TILING(0);
1329                 break;
1330         case 2:
1331                 gb_tiling_config = PIPE_TILING(1);
1332                 break;
1333         case 4:
1334                 gb_tiling_config = PIPE_TILING(2);
1335                 break;
1336         case 8:
1337                 gb_tiling_config = PIPE_TILING(3);
1338                 break;
1339         }
1340         rdev->config.rv770.tiling_npipes = rdev->config.rv770.max_tile_pipes;
1341
1342         disabled_rb_mask = (RREG32(CC_RB_BACKEND_DISABLE) >> 16) & R7XX_MAX_BACKENDS_MASK;
1343         tmp = 0;
1344         for (i = 0; i < rdev->config.rv770.max_backends; i++)
1345                 tmp |= (1 << i);
1346         /* if all the backends are disabled, fix it up here */
1347         if ((disabled_rb_mask & tmp) == tmp) {
1348                 for (i = 0; i < rdev->config.rv770.max_backends; i++)
1349                         disabled_rb_mask &= ~(1 << i);
1350         }
1351         tmp = (gb_tiling_config & PIPE_TILING__MASK) >> PIPE_TILING__SHIFT;
1352         tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.rv770.max_backends,
1353                                         R7XX_MAX_BACKENDS, disabled_rb_mask);
1354         gb_tiling_config |= tmp << 16;
1355         rdev->config.rv770.backend_map = tmp;
1356
1357         if (rdev->family == CHIP_RV770)
1358                 gb_tiling_config |= BANK_TILING(1);
1359         else {
1360                 if ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT)
1361                         gb_tiling_config |= BANK_TILING(1);
1362                 else
1363                         gb_tiling_config |= BANK_TILING(0);
1364         }
1365         rdev->config.rv770.tiling_nbanks = 4 << ((gb_tiling_config >> 4) & 0x3);
1366         gb_tiling_config |= GROUP_SIZE((mc_arb_ramcfg & BURSTLENGTH_MASK) >> BURSTLENGTH_SHIFT);
1367         if (((mc_arb_ramcfg & NOOFROWS_MASK) >> NOOFROWS_SHIFT) > 3) {
1368                 gb_tiling_config |= ROW_TILING(3);
1369                 gb_tiling_config |= SAMPLE_SPLIT(3);
1370         } else {
1371                 gb_tiling_config |=
1372                         ROW_TILING(((mc_arb_ramcfg & NOOFROWS_MASK) >> NOOFROWS_SHIFT));
1373                 gb_tiling_config |=
1374                         SAMPLE_SPLIT(((mc_arb_ramcfg & NOOFROWS_MASK) >> NOOFROWS_SHIFT));
1375         }
1376
1377         gb_tiling_config |= BANK_SWAPS(1);
1378         rdev->config.rv770.tile_config = gb_tiling_config;
1379
1380         WREG32(GB_TILING_CONFIG, gb_tiling_config);
1381         WREG32(DCP_TILING_CONFIG, (gb_tiling_config & 0xffff));
1382         WREG32(HDP_TILING_CONFIG, (gb_tiling_config & 0xffff));
1383         WREG32(DMA_TILING_CONFIG, (gb_tiling_config & 0xffff));
1384         WREG32(DMA_TILING_CONFIG2, (gb_tiling_config & 0xffff));
1385         if (rdev->family == CHIP_RV730) {
1386                 WREG32(UVD_UDEC_DB_TILING_CONFIG, (gb_tiling_config & 0xffff));
1387                 WREG32(UVD_UDEC_DBW_TILING_CONFIG, (gb_tiling_config & 0xffff));
1388                 WREG32(UVD_UDEC_TILING_CONFIG, (gb_tiling_config & 0xffff));
1389         }
1390
1391         WREG32(CGTS_SYS_TCC_DISABLE, 0);
1392         WREG32(CGTS_TCC_DISABLE, 0);
1393         WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
1394         WREG32(CGTS_USER_TCC_DISABLE, 0);
1395
1396
1397         num_qd_pipes = R7XX_MAX_PIPES - r600_count_pipe_bits((cc_gc_shader_pipe_config & INACTIVE_QD_PIPES_MASK) >> 8);
1398         WREG32(VGT_OUT_DEALLOC_CNTL, (num_qd_pipes * 4) & DEALLOC_DIST_MASK);
1399         WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, ((num_qd_pipes * 4) - 2) & VTX_REUSE_DEPTH_MASK);
1400
1401         /* set HW defaults for 3D engine */
1402         WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
1403                                      ROQ_IB2_START(0x2b)));
1404
1405         WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
1406
1407         ta_aux_cntl = RREG32(TA_CNTL_AUX);
1408         WREG32(TA_CNTL_AUX, ta_aux_cntl | DISABLE_CUBE_ANISO);
1409
1410         sx_debug_1 = RREG32(SX_DEBUG_1);
1411         sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
1412         WREG32(SX_DEBUG_1, sx_debug_1);
1413
1414         smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
1415         smx_dc_ctl0 &= ~CACHE_DEPTH(0x1ff);
1416         smx_dc_ctl0 |= CACHE_DEPTH((rdev->config.rv770.sx_num_of_sets * 64) - 1);
1417         WREG32(SMX_DC_CTL0, smx_dc_ctl0);
1418
1419         if (rdev->family != CHIP_RV740)
1420                 WREG32(SMX_EVENT_CTL, (ES_FLUSH_CTL(4) |
1421                                        GS_FLUSH_CTL(4) |
1422                                        ACK_FLUSH_CTL(3) |
1423                                        SYNC_FLUSH_CTL));
1424
1425         if (rdev->family != CHIP_RV770)
1426                 WREG32(SMX_SAR_CTL0, 0x00003f3f);
1427
1428         db_debug3 = RREG32(DB_DEBUG3);
1429         db_debug3 &= ~DB_CLK_OFF_DELAY(0x1f);
1430         switch (rdev->family) {
1431         case CHIP_RV770:
1432         case CHIP_RV740:
1433                 db_debug3 |= DB_CLK_OFF_DELAY(0x1f);
1434                 break;
1435         case CHIP_RV710:
1436         case CHIP_RV730:
1437         default:
1438                 db_debug3 |= DB_CLK_OFF_DELAY(2);
1439                 break;
1440         }
1441         WREG32(DB_DEBUG3, db_debug3);
1442
1443         if (rdev->family != CHIP_RV770) {
1444                 db_debug4 = RREG32(DB_DEBUG4);
1445                 db_debug4 |= DISABLE_TILE_COVERED_FOR_PS_ITER;
1446                 WREG32(DB_DEBUG4, db_debug4);
1447         }
1448
1449         WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.rv770.sx_max_export_size / 4) - 1) |
1450                                         POSITION_BUFFER_SIZE((rdev->config.rv770.sx_max_export_pos_size / 4) - 1) |
1451                                         SMX_BUFFER_SIZE((rdev->config.rv770.sx_max_export_smx_size / 4) - 1)));
1452
1453         WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.rv770.sc_prim_fifo_size) |
1454                                  SC_HIZ_TILE_FIFO_SIZE(rdev->config.rv770.sc_hiz_tile_fifo_size) |
1455                                  SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.rv770.sc_earlyz_tile_fifo_fize)));
1456
1457         WREG32(PA_SC_MULTI_CHIP_CNTL, 0);
1458
1459         WREG32(VGT_NUM_INSTANCES, 1);
1460
1461         WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
1462
1463         WREG32(CP_PERFMON_CNTL, 0);
1464
1465         sq_ms_fifo_sizes = (CACHE_FIFO_SIZE(16 * rdev->config.rv770.sq_num_cf_insts) |
1466                             DONE_FIFO_HIWATER(0xe0) |
1467                             ALU_UPDATE_FIFO_HIWATER(0x8));
1468         switch (rdev->family) {
1469         case CHIP_RV770:
1470         case CHIP_RV730:
1471         case CHIP_RV710:
1472                 sq_ms_fifo_sizes |= FETCH_FIFO_HIWATER(0x1);
1473                 break;
1474         case CHIP_RV740:
1475         default:
1476                 sq_ms_fifo_sizes |= FETCH_FIFO_HIWATER(0x4);
1477                 break;
1478         }
1479         WREG32(SQ_MS_FIFO_SIZES, sq_ms_fifo_sizes);
1480
1481         /* SQ_CONFIG, SQ_GPR_RESOURCE_MGMT, SQ_THREAD_RESOURCE_MGMT, SQ_STACK_RESOURCE_MGMT
1482          * should be adjusted as needed by the 2D/3D drivers.  This just sets default values
1483          */
1484         sq_config = RREG32(SQ_CONFIG);
1485         sq_config &= ~(PS_PRIO(3) |
1486                        VS_PRIO(3) |
1487                        GS_PRIO(3) |
1488                        ES_PRIO(3));
1489         sq_config |= (DX9_CONSTS |
1490                       VC_ENABLE |
1491                       EXPORT_SRC_C |
1492                       PS_PRIO(0) |
1493                       VS_PRIO(1) |
1494                       GS_PRIO(2) |
1495                       ES_PRIO(3));
1496         if (rdev->family == CHIP_RV710)
1497                 /* no vertex cache */
1498                 sq_config &= ~VC_ENABLE;
1499
1500         WREG32(SQ_CONFIG, sq_config);
1501
1502         WREG32(SQ_GPR_RESOURCE_MGMT_1,  (NUM_PS_GPRS((rdev->config.rv770.max_gprs * 24)/64) |
1503                                          NUM_VS_GPRS((rdev->config.rv770.max_gprs * 24)/64) |
1504                                          NUM_CLAUSE_TEMP_GPRS(((rdev->config.rv770.max_gprs * 24)/64)/2)));
1505
1506         WREG32(SQ_GPR_RESOURCE_MGMT_2,  (NUM_GS_GPRS((rdev->config.rv770.max_gprs * 7)/64) |
1507                                          NUM_ES_GPRS((rdev->config.rv770.max_gprs * 7)/64)));
1508
1509         sq_thread_resource_mgmt = (NUM_PS_THREADS((rdev->config.rv770.max_threads * 4)/8) |
1510                                    NUM_VS_THREADS((rdev->config.rv770.max_threads * 2)/8) |
1511                                    NUM_ES_THREADS((rdev->config.rv770.max_threads * 1)/8));
1512         if (((rdev->config.rv770.max_threads * 1) / 8) > rdev->config.rv770.max_gs_threads)
1513                 sq_thread_resource_mgmt |= NUM_GS_THREADS(rdev->config.rv770.max_gs_threads);
1514         else
1515                 sq_thread_resource_mgmt |= NUM_GS_THREADS((rdev->config.rv770.max_gs_threads * 1)/8);
1516         WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
1517
1518         WREG32(SQ_STACK_RESOURCE_MGMT_1, (NUM_PS_STACK_ENTRIES((rdev->config.rv770.max_stack_entries * 1)/4) |
1519                                                      NUM_VS_STACK_ENTRIES((rdev->config.rv770.max_stack_entries * 1)/4)));
1520
1521         WREG32(SQ_STACK_RESOURCE_MGMT_2, (NUM_GS_STACK_ENTRIES((rdev->config.rv770.max_stack_entries * 1)/4) |
1522                                                      NUM_ES_STACK_ENTRIES((rdev->config.rv770.max_stack_entries * 1)/4)));
1523
1524         sq_dyn_gpr_size_simd_ab_0 = (SIMDA_RING0((rdev->config.rv770.max_gprs * 38)/64) |
1525                                      SIMDA_RING1((rdev->config.rv770.max_gprs * 38)/64) |
1526                                      SIMDB_RING0((rdev->config.rv770.max_gprs * 38)/64) |
1527                                      SIMDB_RING1((rdev->config.rv770.max_gprs * 38)/64));
1528
1529         WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_0, sq_dyn_gpr_size_simd_ab_0);
1530         WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_1, sq_dyn_gpr_size_simd_ab_0);
1531         WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_2, sq_dyn_gpr_size_simd_ab_0);
1532         WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_3, sq_dyn_gpr_size_simd_ab_0);
1533         WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_4, sq_dyn_gpr_size_simd_ab_0);
1534         WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_5, sq_dyn_gpr_size_simd_ab_0);
1535         WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_6, sq_dyn_gpr_size_simd_ab_0);
1536         WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_7, sq_dyn_gpr_size_simd_ab_0);
1537
1538         WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
1539                                           FORCE_EOV_MAX_REZ_CNT(255)));
1540
1541         if (rdev->family == CHIP_RV710)
1542                 WREG32(VGT_CACHE_INVALIDATION, (CACHE_INVALIDATION(TC_ONLY) |
1543                                                 AUTO_INVLD_EN(ES_AND_GS_AUTO)));
1544         else
1545                 WREG32(VGT_CACHE_INVALIDATION, (CACHE_INVALIDATION(VC_AND_TC) |
1546                                                 AUTO_INVLD_EN(ES_AND_GS_AUTO)));
1547
1548         switch (rdev->family) {
1549         case CHIP_RV770:
1550         case CHIP_RV730:
1551         case CHIP_RV740:
1552                 gs_prim_buffer_depth = 384;
1553                 break;
1554         case CHIP_RV710:
1555                 gs_prim_buffer_depth = 128;
1556                 break;
1557         default:
1558                 break;
1559         }
1560
1561         num_gs_verts_per_thread = rdev->config.rv770.max_pipes * 16;
1562         vgt_gs_per_es = gs_prim_buffer_depth + num_gs_verts_per_thread;
1563         /* Max value for this is 256 */
1564         if (vgt_gs_per_es > 256)
1565                 vgt_gs_per_es = 256;
1566
1567         WREG32(VGT_ES_PER_GS, 128);
1568         WREG32(VGT_GS_PER_ES, vgt_gs_per_es);
1569         WREG32(VGT_GS_PER_VS, 2);
1570
1571         /* more default values. 2D/3D driver should adjust as needed */
1572         WREG32(VGT_GS_VERTEX_REUSE, 16);
1573         WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
1574         WREG32(VGT_STRMOUT_EN, 0);
1575         WREG32(SX_MISC, 0);
1576         WREG32(PA_SC_MODE_CNTL, 0);
1577         WREG32(PA_SC_EDGERULE, 0xaaaaaaaa);
1578         WREG32(PA_SC_AA_CONFIG, 0);
1579         WREG32(PA_SC_CLIPRECT_RULE, 0xffff);
1580         WREG32(PA_SC_LINE_STIPPLE, 0);
1581         WREG32(SPI_INPUT_Z, 0);
1582         WREG32(SPI_PS_IN_CONTROL_0, NUM_INTERP(2));
1583         WREG32(CB_COLOR7_FRAG, 0);
1584
1585         /* clear render buffer base addresses */
1586         WREG32(CB_COLOR0_BASE, 0);
1587         WREG32(CB_COLOR1_BASE, 0);
1588         WREG32(CB_COLOR2_BASE, 0);
1589         WREG32(CB_COLOR3_BASE, 0);
1590         WREG32(CB_COLOR4_BASE, 0);
1591         WREG32(CB_COLOR5_BASE, 0);
1592         WREG32(CB_COLOR6_BASE, 0);
1593         WREG32(CB_COLOR7_BASE, 0);
1594
1595         WREG32(TCP_CNTL, 0);
1596
1597         hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
1598         WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
1599
1600         WREG32(PA_SC_MULTI_CHIP_CNTL, 0);
1601
1602         WREG32(PA_CL_ENHANCE, (CLIP_VTX_REORDER_ENA |
1603                                           NUM_CLIP_SEQ(3)));
1604         WREG32(VC_ENHANCE, 0);
1605 }
1606
1607 void r700_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc)
1608 {
1609         u64 size_bf, size_af;
1610
1611         if (mc->mc_vram_size > 0xE0000000) {
1612                 /* leave room for at least 512M GTT */
1613                 dev_warn(rdev->dev, "limiting VRAM\n");
1614                 mc->real_vram_size = 0xE0000000;
1615                 mc->mc_vram_size = 0xE0000000;
1616         }
1617         if (rdev->flags & RADEON_IS_AGP) {
1618                 size_bf = mc->gtt_start;
1619                 size_af = mc->mc_mask - mc->gtt_end;
1620                 if (size_bf > size_af) {
1621                         if (mc->mc_vram_size > size_bf) {
1622                                 dev_warn(rdev->dev, "limiting VRAM\n");
1623                                 mc->real_vram_size = size_bf;
1624                                 mc->mc_vram_size = size_bf;
1625                         }
1626                         mc->vram_start = mc->gtt_start - mc->mc_vram_size;
1627                 } else {
1628                         if (mc->mc_vram_size > size_af) {
1629                                 dev_warn(rdev->dev, "limiting VRAM\n");
1630                                 mc->real_vram_size = size_af;
1631                                 mc->mc_vram_size = size_af;
1632                         }
1633                         mc->vram_start = mc->gtt_end + 1;
1634                 }
1635                 mc->vram_end = mc->vram_start + mc->mc_vram_size - 1;
1636                 dev_info(rdev->dev, "VRAM: %lluM 0x%08llX - 0x%08llX (%lluM used)\n",
1637                                 mc->mc_vram_size >> 20, mc->vram_start,
1638                                 mc->vram_end, mc->real_vram_size >> 20);
1639         } else {
1640                 radeon_vram_location(rdev, &rdev->mc, 0);
1641                 rdev->mc.gtt_base_align = 0;
1642                 radeon_gtt_location(rdev, mc);
1643         }
1644 }
1645
1646 static int rv770_mc_init(struct radeon_device *rdev)
1647 {
1648         u32 tmp;
1649         int chansize, numchan;
1650
1651         /* Get VRAM informations */
1652         rdev->mc.vram_is_ddr = true;
1653         tmp = RREG32(MC_ARB_RAMCFG);
1654         if (tmp & CHANSIZE_OVERRIDE) {
1655                 chansize = 16;
1656         } else if (tmp & CHANSIZE_MASK) {
1657                 chansize = 64;
1658         } else {
1659                 chansize = 32;
1660         }
1661         tmp = RREG32(MC_SHARED_CHMAP);
1662         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1663         case 0:
1664         default:
1665                 numchan = 1;
1666                 break;
1667         case 1:
1668                 numchan = 2;
1669                 break;
1670         case 2:
1671                 numchan = 4;
1672                 break;
1673         case 3:
1674                 numchan = 8;
1675                 break;
1676         }
1677         rdev->mc.vram_width = numchan * chansize;
1678         /* Could aper size report 0 ? */
1679         rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
1680         rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
1681         /* Setup GPU memory space */
1682         rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
1683         rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
1684         rdev->mc.visible_vram_size = rdev->mc.aper_size;
1685         r700_vram_gtt_location(rdev, &rdev->mc);
1686         radeon_update_bandwidth_info(rdev);
1687
1688         return 0;
1689 }
1690
1691 static void rv770_uvd_init(struct radeon_device *rdev)
1692 {
1693         int r;
1694
1695         if (!rdev->has_uvd)
1696                 return;
1697
1698         r = radeon_uvd_init(rdev);
1699         if (r) {
1700                 dev_err(rdev->dev, "failed UVD (%d) init.\n", r);
1701                 /*
1702                  * At this point rdev->uvd.vcpu_bo is NULL which trickles down
1703                  * to early fails uvd_v2_2_resume() and thus nothing happens
1704                  * there. So it is pointless to try to go through that code
1705                  * hence why we disable uvd here.
1706                  */
1707                 rdev->has_uvd = false;
1708                 return;
1709         }
1710         rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
1711         r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX], 4096);
1712 }
1713
1714 static void rv770_uvd_start(struct radeon_device *rdev)
1715 {
1716         int r;
1717
1718         if (!rdev->has_uvd)
1719                 return;
1720
1721         r = uvd_v2_2_resume(rdev);
1722         if (r) {
1723                 dev_err(rdev->dev, "failed UVD resume (%d).\n", r);
1724                 goto error;
1725         }
1726         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_UVD_INDEX);
1727         if (r) {
1728                 dev_err(rdev->dev, "failed initializing UVD fences (%d).\n", r);
1729                 goto error;
1730         }
1731         return;
1732
1733 error:
1734         rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
1735 }
1736
1737 static void rv770_uvd_resume(struct radeon_device *rdev)
1738 {
1739         struct radeon_ring *ring;
1740         int r;
1741
1742         if (!rdev->has_uvd || !rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size)
1743                 return;
1744
1745         ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
1746         r = radeon_ring_init(rdev, ring, ring->ring_size, 0, PACKET0(UVD_NO_OP, 0));
1747         if (r) {
1748                 dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r);
1749                 return;
1750         }
1751         r = uvd_v1_0_init(rdev);
1752         if (r) {
1753                 dev_err(rdev->dev, "failed initializing UVD (%d).\n", r);
1754                 return;
1755         }
1756 }
1757
1758 static int rv770_startup(struct radeon_device *rdev)
1759 {
1760         struct radeon_ring *ring;
1761         int r;
1762
1763         /* enable pcie gen2 link */
1764         rv770_pcie_gen2_enable(rdev);
1765
1766         /* scratch needs to be initialized before MC */
1767         r = r600_vram_scratch_init(rdev);
1768         if (r)
1769                 return r;
1770
1771         rv770_mc_program(rdev);
1772
1773         if (rdev->flags & RADEON_IS_AGP) {
1774                 rv770_agp_enable(rdev);
1775         } else {
1776                 r = rv770_pcie_gart_enable(rdev);
1777                 if (r)
1778                         return r;
1779         }
1780
1781         rv770_gpu_init(rdev);
1782
1783         /* allocate wb buffer */
1784         r = radeon_wb_init(rdev);
1785         if (r)
1786                 return r;
1787
1788         r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
1789         if (r) {
1790                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
1791                 return r;
1792         }
1793
1794         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
1795         if (r) {
1796                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
1797                 return r;
1798         }
1799
1800         rv770_uvd_start(rdev);
1801
1802         /* Enable IRQ */
1803         if (!rdev->irq.installed) {
1804                 r = radeon_irq_kms_init(rdev);
1805                 if (r)
1806                         return r;
1807         }
1808
1809         r = r600_irq_init(rdev);
1810         if (r) {
1811                 DRM_ERROR("radeon: IH init failed (%d).\n", r);
1812                 radeon_irq_kms_fini(rdev);
1813                 return r;
1814         }
1815         r600_irq_set(rdev);
1816
1817         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1818         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
1819                              RADEON_CP_PACKET2);
1820         if (r)
1821                 return r;
1822
1823         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
1824         r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
1825                              DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
1826         if (r)
1827                 return r;
1828
1829         r = rv770_cp_load_microcode(rdev);
1830         if (r)
1831                 return r;
1832         r = r600_cp_resume(rdev);
1833         if (r)
1834                 return r;
1835
1836         r = r600_dma_resume(rdev);
1837         if (r)
1838                 return r;
1839
1840         rv770_uvd_resume(rdev);
1841
1842         r = radeon_ib_pool_init(rdev);
1843         if (r) {
1844                 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
1845                 return r;
1846         }
1847
1848         r = radeon_audio_init(rdev);
1849         if (r) {
1850                 DRM_ERROR("radeon: audio init failed\n");
1851                 return r;
1852         }
1853
1854         return 0;
1855 }
1856
1857 int rv770_resume(struct radeon_device *rdev)
1858 {
1859         int r;
1860
1861         /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
1862          * posting will perform necessary task to bring back GPU into good
1863          * shape.
1864          */
1865         /* post card */
1866         atom_asic_init(rdev->mode_info.atom_context);
1867
1868         /* init golden registers */
1869         rv770_init_golden_registers(rdev);
1870
1871         if (rdev->pm.pm_method == PM_METHOD_DPM)
1872                 radeon_pm_resume(rdev);
1873
1874         rdev->accel_working = true;
1875         r = rv770_startup(rdev);
1876         if (r) {
1877                 DRM_ERROR("r600 startup failed on resume\n");
1878                 rdev->accel_working = false;
1879                 return r;
1880         }
1881
1882         return r;
1883
1884 }
1885
1886 int rv770_suspend(struct radeon_device *rdev)
1887 {
1888         radeon_pm_suspend(rdev);
1889         radeon_audio_fini(rdev);
1890         if (rdev->has_uvd) {
1891                 uvd_v1_0_fini(rdev);
1892                 radeon_uvd_suspend(rdev);
1893         }
1894         r700_cp_stop(rdev);
1895         r600_dma_stop(rdev);
1896         r600_irq_suspend(rdev);
1897         radeon_wb_disable(rdev);
1898         rv770_pcie_gart_disable(rdev);
1899
1900         return 0;
1901 }
1902
1903 /* Plan is to move initialization in that function and use
1904  * helper function so that radeon_device_init pretty much
1905  * do nothing more than calling asic specific function. This
1906  * should also allow to remove a bunch of callback function
1907  * like vram_info.
1908  */
1909 int rv770_init(struct radeon_device *rdev)
1910 {
1911         int r;
1912
1913         /* Read BIOS */
1914         if (!radeon_get_bios(rdev)) {
1915                 if (ASIC_IS_AVIVO(rdev))
1916                         return -EINVAL;
1917         }
1918         /* Must be an ATOMBIOS */
1919         if (!rdev->is_atom_bios) {
1920                 dev_err(rdev->dev, "Expecting atombios for R600 GPU\n");
1921                 return -EINVAL;
1922         }
1923         r = radeon_atombios_init(rdev);
1924         if (r)
1925                 return r;
1926         /* Post card if necessary */
1927         if (!radeon_card_posted(rdev)) {
1928                 if (!rdev->bios) {
1929                         dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
1930                         return -EINVAL;
1931                 }
1932                 DRM_INFO("GPU not posted. posting now...\n");
1933                 atom_asic_init(rdev->mode_info.atom_context);
1934         }
1935         /* init golden registers */
1936         rv770_init_golden_registers(rdev);
1937         /* Initialize scratch registers */
1938         r600_scratch_init(rdev);
1939         /* Initialize surface registers */
1940         radeon_surface_init(rdev);
1941         /* Initialize clocks */
1942         radeon_get_clock_info(rdev->ddev);
1943         /* Fence driver */
1944         radeon_fence_driver_init(rdev);
1945         /* initialize AGP */
1946         if (rdev->flags & RADEON_IS_AGP) {
1947                 r = radeon_agp_init(rdev);
1948                 if (r)
1949                         radeon_agp_disable(rdev);
1950         }
1951         r = rv770_mc_init(rdev);
1952         if (r)
1953                 return r;
1954         /* Memory manager */
1955         r = radeon_bo_init(rdev);
1956         if (r)
1957                 return r;
1958
1959         if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
1960                 r = r600_init_microcode(rdev);
1961                 if (r) {
1962                         DRM_ERROR("Failed to load firmware!\n");
1963                         return r;
1964                 }
1965         }
1966
1967         /* Initialize power management */
1968         radeon_pm_init(rdev);
1969
1970         rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
1971         r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
1972
1973         rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
1974         r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
1975
1976         rv770_uvd_init(rdev);
1977
1978         rdev->ih.ring_obj = NULL;
1979         r600_ih_ring_init(rdev, 64 * 1024);
1980
1981         r = r600_pcie_gart_init(rdev);
1982         if (r)
1983                 return r;
1984
1985         rdev->accel_working = true;
1986         r = rv770_startup(rdev);
1987         if (r) {
1988                 dev_err(rdev->dev, "disabling GPU acceleration\n");
1989                 r700_cp_fini(rdev);
1990                 r600_dma_fini(rdev);
1991                 r600_irq_fini(rdev);
1992                 radeon_wb_fini(rdev);
1993                 radeon_ib_pool_fini(rdev);
1994                 radeon_irq_kms_fini(rdev);
1995                 rv770_pcie_gart_fini(rdev);
1996                 rdev->accel_working = false;
1997         }
1998
1999         return 0;
2000 }
2001
2002 void rv770_fini(struct radeon_device *rdev)
2003 {
2004         radeon_pm_fini(rdev);
2005         r700_cp_fini(rdev);
2006         r600_dma_fini(rdev);
2007         r600_irq_fini(rdev);
2008         radeon_wb_fini(rdev);
2009         radeon_ib_pool_fini(rdev);
2010         radeon_irq_kms_fini(rdev);
2011         uvd_v1_0_fini(rdev);
2012         radeon_uvd_fini(rdev);
2013         rv770_pcie_gart_fini(rdev);
2014         r600_vram_scratch_fini(rdev);
2015         radeon_gem_fini(rdev);
2016         radeon_fence_driver_fini(rdev);
2017         radeon_agp_fini(rdev);
2018         radeon_bo_fini(rdev);
2019         radeon_atombios_fini(rdev);
2020         kfree(rdev->bios);
2021         rdev->bios = NULL;
2022 }
2023
2024 static void rv770_pcie_gen2_enable(struct radeon_device *rdev)
2025 {
2026         u32 link_width_cntl, lanes, speed_cntl, tmp;
2027         u16 link_cntl2;
2028
2029         if (radeon_pcie_gen2 == 0)
2030                 return;
2031
2032         if (rdev->flags & RADEON_IS_IGP)
2033                 return;
2034
2035         if (!(rdev->flags & RADEON_IS_PCIE))
2036                 return;
2037
2038         /* x2 cards have a special sequence */
2039         if (ASIC_IS_X2(rdev))
2040                 return;
2041
2042         if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
2043                 (rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
2044                 return;
2045
2046         DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
2047
2048         /* advertise upconfig capability */
2049         link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
2050         link_width_cntl &= ~LC_UPCONFIGURE_DIS;
2051         WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
2052         link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
2053         if (link_width_cntl & LC_RENEGOTIATION_SUPPORT) {
2054                 lanes = (link_width_cntl & LC_LINK_WIDTH_RD_MASK) >> LC_LINK_WIDTH_RD_SHIFT;
2055                 link_width_cntl &= ~(LC_LINK_WIDTH_MASK |
2056                                      LC_RECONFIG_ARC_MISSING_ESCAPE);
2057                 link_width_cntl |= lanes | LC_RECONFIG_NOW |
2058                         LC_RENEGOTIATE_EN | LC_UPCONFIGURE_SUPPORT;
2059                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
2060         } else {
2061                 link_width_cntl |= LC_UPCONFIGURE_DIS;
2062                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
2063         }
2064
2065         speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
2066         if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) &&
2067             (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
2068
2069                 tmp = RREG32(0x541c);
2070                 WREG32(0x541c, tmp | 0x8);
2071                 WREG32(MM_CFGREGS_CNTL, MM_WR_TO_CFG_EN);
2072                 link_cntl2 = RREG16(0x4088);
2073                 link_cntl2 &= ~TARGET_LINK_SPEED_MASK;
2074                 link_cntl2 |= 0x2;
2075                 WREG16(0x4088, link_cntl2);
2076                 WREG32(MM_CFGREGS_CNTL, 0);
2077
2078                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
2079                 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
2080                 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
2081
2082                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
2083                 speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
2084                 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
2085
2086                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
2087                 speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
2088                 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
2089
2090                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
2091                 speed_cntl |= LC_GEN2_EN_STRAP;
2092                 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
2093
2094         } else {
2095                 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
2096                 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
2097                 if (1)
2098                         link_width_cntl |= LC_UPCONFIGURE_DIS;
2099                 else
2100                         link_width_cntl &= ~LC_UPCONFIGURE_DIS;
2101                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
2102         }
2103 }