drivers/gpu/drm/amd/amdgpu/amdgpu_device.c

   1 /*
   2  * Copyright 2008 Advanced Micro Devices, Inc.
   3  * Copyright 2008 Red Hat Inc.
   4  * Copyright 2009 Jerome Glisse.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22  * OTHER DEALINGS IN THE SOFTWARE.
  23  *
  24  * Authors: Dave Airlie
  25  *          Alex Deucher
  26  *          Jerome Glisse
  27  */
  28 #include <linux/console.h>
  29 #include <linux/slab.h>
  30 #include <linux/debugfs.h>
  31 #include <drm/drmP.h>
  32 #include <drm/drm_crtc_helper.h>
  33 #include <drm/amdgpu_drm.h>
  34 #include <linux/vgaarb.h>
  35 #include <linux/vga_switcheroo.h>
  36 #include <linux/efi.h>
  37 #include "amdgpu.h"
  38 #include "amdgpu_i2c.h"
  39 #include "atom.h"
  40 #include "amdgpu_atombios.h"
  41 #ifdef CONFIG_DRM_AMDGPU_CIK
  42 #include "cik.h"
  43 #endif
  44 #include "vi.h"
  45 #include "bif/bif_4_1_d.h"
  46
  47 static int amdgpu_debugfs_regs_init(struct amdgpu_device *adev);
  48 static void amdgpu_debugfs_regs_cleanup(struct amdgpu_device *adev);
  49
  50 static const char *amdgpu_asic_name[] = {
  51         "BONAIRE",
  52         "KAVERI",
  53         "KABINI",
  54         "HAWAII",
  55         "MULLINS",
  56         "TOPAZ",
  57         "TONGA",
  58         "FIJI",
  59         "CARRIZO",
  60         "LAST",
  61 };
  62
  63 bool amdgpu_device_is_px(struct drm_device *dev)
  64 {
  65         struct amdgpu_device *adev = dev->dev_private;
  66
  67         if (adev->flags & AMD_IS_PX)
  68                 return true;
  69         return false;
  70 }
  71
  72 /*
  73  * MMIO register access helper functions.
  74  */
  75 uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg,
  76                         bool always_indirect)
  77 {
  78         if ((reg * 4) < adev->rmmio_size && !always_indirect)
  79                 return readl(((void __iomem *)adev->rmmio) + (reg * 4));
  80         else {
  81                 unsigned long flags;
  82                 uint32_t ret;
  83
  84                 spin_lock_irqsave(&adev->mmio_idx_lock, flags);
  85                 writel((reg * 4), ((void __iomem *)adev->rmmio) + (mmMM_INDEX * 4));
  86                 ret = readl(((void __iomem *)adev->rmmio) + (mmMM_DATA * 4));
  87                 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
  88
  89                 return ret;
  90         }
  91 }
  92
  93 void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v,
  94                     bool always_indirect)
  95 {
  96         if ((reg * 4) < adev->rmmio_size && !always_indirect)
  97                 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
  98         else {
  99                 unsigned long flags;
 100
 101                 spin_lock_irqsave(&adev->mmio_idx_lock, flags);
 102                 writel((reg * 4), ((void __iomem *)adev->rmmio) + (mmMM_INDEX * 4));
 103                 writel(v, ((void __iomem *)adev->rmmio) + (mmMM_DATA * 4));
 104                 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
 105         }
 106 }
 107
 108 u32 amdgpu_io_rreg(struct amdgpu_device *adev, u32 reg)
 109 {
 110         if ((reg * 4) < adev->rio_mem_size)
 111                 return ioread32(adev->rio_mem + (reg * 4));
 112         else {
 113                 iowrite32((reg * 4), adev->rio_mem + (mmMM_INDEX * 4));
 114                 return ioread32(adev->rio_mem + (mmMM_DATA * 4));
 115         }
 116 }
 117
 118 void amdgpu_io_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 119 {
 120
 121         if ((reg * 4) < adev->rio_mem_size)
 122                 iowrite32(v, adev->rio_mem + (reg * 4));
 123         else {
 124                 iowrite32((reg * 4), adev->rio_mem + (mmMM_INDEX * 4));
 125                 iowrite32(v, adev->rio_mem + (mmMM_DATA * 4));
 126         }
 127 }
 128
 129 /**
 130  * amdgpu_mm_rdoorbell - read a doorbell dword
 131  *
 132  * @adev: amdgpu_device pointer
 133  * @index: doorbell index
 134  *
 135  * Returns the value in the doorbell aperture at the
 136  * requested doorbell index (CIK).
 137  */
 138 u32 amdgpu_mm_rdoorbell(struct amdgpu_device *adev, u32 index)
 139 {
 140         if (index < adev->doorbell.num_doorbells) {
 141                 return readl(adev->doorbell.ptr + index);
 142         } else {
 143                 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
 144                 return 0;
 145         }
 146 }
 147
 148 /**
 149  * amdgpu_mm_wdoorbell - write a doorbell dword
 150  *
 151  * @adev: amdgpu_device pointer
 152  * @index: doorbell index
 153  * @v: value to write
 154  *
 155  * Writes @v to the doorbell aperture at the
 156  * requested doorbell index (CIK).
 157  */
 158 void amdgpu_mm_wdoorbell(struct amdgpu_device *adev, u32 index, u32 v)
 159 {
 160         if (index < adev->doorbell.num_doorbells) {
 161                 writel(v, adev->doorbell.ptr + index);
 162         } else {
 163                 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
 164         }
 165 }
 166
 167 /**
 168  * amdgpu_invalid_rreg - dummy reg read function
 169  *
 170  * @adev: amdgpu device pointer
 171  * @reg: offset of register
 172  *
 173  * Dummy register read function.  Used for register blocks
 174  * that certain asics don't have (all asics).
 175  * Returns the value in the register.
 176  */
 177 static uint32_t amdgpu_invalid_rreg(struct amdgpu_device *adev, uint32_t reg)
 178 {
 179         DRM_ERROR("Invalid callback to read register 0x%04X\n", reg);
 180         BUG();
 181         return 0;
 182 }
 183
 184 /**
 185  * amdgpu_invalid_wreg - dummy reg write function
 186  *
 187  * @adev: amdgpu device pointer
 188  * @reg: offset of register
 189  * @v: value to write to the register
 190  *
 191  * Dummy register read function.  Used for register blocks
 192  * that certain asics don't have (all asics).
 193  */
 194 static void amdgpu_invalid_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
 195 {
 196         DRM_ERROR("Invalid callback to write register 0x%04X with 0x%08X\n",
 197                   reg, v);
 198         BUG();
 199 }
 200
 201 /**
 202  * amdgpu_block_invalid_rreg - dummy reg read function
 203  *
 204  * @adev: amdgpu device pointer
 205  * @block: offset of instance
 206  * @reg: offset of register
 207  *
 208  * Dummy register read function.  Used for register blocks
 209  * that certain asics don't have (all asics).
 210  * Returns the value in the register.
 211  */
 212 static uint32_t amdgpu_block_invalid_rreg(struct amdgpu_device *adev,
 213                                           uint32_t block, uint32_t reg)
 214 {
 215         DRM_ERROR("Invalid callback to read register 0x%04X in block 0x%04X\n",
 216                   reg, block);
 217         BUG();
 218         return 0;
 219 }
 220
 221 /**
 222  * amdgpu_block_invalid_wreg - dummy reg write function
 223  *
 224  * @adev: amdgpu device pointer
 225  * @block: offset of instance
 226  * @reg: offset of register
 227  * @v: value to write to the register
 228  *
 229  * Dummy register read function.  Used for register blocks
 230  * that certain asics don't have (all asics).
 231  */
 232 static void amdgpu_block_invalid_wreg(struct amdgpu_device *adev,
 233                                       uint32_t block,
 234                                       uint32_t reg, uint32_t v)
 235 {
 236         DRM_ERROR("Invalid block callback to write register 0x%04X in block 0x%04X with 0x%08X\n",
 237                   reg, block, v);
 238         BUG();
 239 }
 240
 241 static int amdgpu_vram_scratch_init(struct amdgpu_device *adev)
 242 {
 243         int r;
 244
 245         if (adev->vram_scratch.robj == NULL) {
 246                 r = amdgpu_bo_create(adev, AMDGPU_GPU_PAGE_SIZE,
 247                                      PAGE_SIZE, true, AMDGPU_GEM_DOMAIN_VRAM,
 248                                      AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
 249                                      NULL, NULL, &adev->vram_scratch.robj);
 250                 if (r) {
 251                         return r;
 252                 }
 253         }
 254
 255         r = amdgpu_bo_reserve(adev->vram_scratch.robj, false);
 256         if (unlikely(r != 0))
 257                 return r;
 258         r = amdgpu_bo_pin(adev->vram_scratch.robj,
 259                           AMDGPU_GEM_DOMAIN_VRAM, &adev->vram_scratch.gpu_addr);
 260         if (r) {
 261                 amdgpu_bo_unreserve(adev->vram_scratch.robj);
 262                 return r;
 263         }
 264         r = amdgpu_bo_kmap(adev->vram_scratch.robj,
 265                                 (void **)&adev->vram_scratch.ptr);
 266         if (r)
 267                 amdgpu_bo_unpin(adev->vram_scratch.robj);
 268         amdgpu_bo_unreserve(adev->vram_scratch.robj);
 269
 270         return r;
 271 }
 272
 273 static void amdgpu_vram_scratch_fini(struct amdgpu_device *adev)
 274 {
 275         int r;
 276
 277         if (adev->vram_scratch.robj == NULL) {
 278                 return;
 279         }
 280         r = amdgpu_bo_reserve(adev->vram_scratch.robj, false);
 281         if (likely(r == 0)) {
 282                 amdgpu_bo_kunmap(adev->vram_scratch.robj);
 283                 amdgpu_bo_unpin(adev->vram_scratch.robj);
 284                 amdgpu_bo_unreserve(adev->vram_scratch.robj);
 285         }
 286         amdgpu_bo_unref(&adev->vram_scratch.robj);
 287 }
 288
 289 /**
 290  * amdgpu_program_register_sequence - program an array of registers.
 291  *
 292  * @adev: amdgpu_device pointer
 293  * @registers: pointer to the register array
 294  * @array_size: size of the register array
 295  *
 296  * Programs an array or registers with and and or masks.
 297  * This is a helper for setting golden registers.
 298  */
 299 void amdgpu_program_register_sequence(struct amdgpu_device *adev,
 300                                       const u32 *registers,
 301                                       const u32 array_size)
 302 {
 303         u32 tmp, reg, and_mask, or_mask;
 304         int i;
 305
 306         if (array_size % 3)
 307                 return;
 308
 309         for (i = 0; i < array_size; i +=3) {
 310                 reg = registers[i + 0];
 311                 and_mask = registers[i + 1];
 312                 or_mask = registers[i + 2];
 313
 314                 if (and_mask == 0xffffffff) {
 315                         tmp = or_mask;
 316                 } else {
 317                         tmp = RREG32(reg);
 318                         tmp &= ~and_mask;
 319                         tmp |= or_mask;
 320                 }
 321                 WREG32(reg, tmp);
 322         }
 323 }
 324
 325 void amdgpu_pci_config_reset(struct amdgpu_device *adev)
 326 {
 327         pci_write_config_dword(adev->pdev, 0x7c, AMDGPU_ASIC_RESET_DATA);
 328 }
 329
 330 /*
 331  * GPU doorbell aperture helpers function.
 332  */
 333 /**
 334  * amdgpu_doorbell_init - Init doorbell driver information.
 335  *
 336  * @adev: amdgpu_device pointer
 337  *
 338  * Init doorbell driver information (CIK)
 339  * Returns 0 on success, error on failure.
 340  */
 341 static int amdgpu_doorbell_init(struct amdgpu_device *adev)
 342 {
 343         /* doorbell bar mapping */
 344         adev->doorbell.base = pci_resource_start(adev->pdev, 2);
 345         adev->doorbell.size = pci_resource_len(adev->pdev, 2);
 346
 347         adev->doorbell.num_doorbells = min_t(u32, adev->doorbell.size / sizeof(u32),
 348                                              AMDGPU_DOORBELL_MAX_ASSIGNMENT+1);
 349         if (adev->doorbell.num_doorbells == 0)
 350                 return -EINVAL;
 351
 352         adev->doorbell.ptr = ioremap(adev->doorbell.base, adev->doorbell.num_doorbells * sizeof(u32));
 353         if (adev->doorbell.ptr == NULL) {
 354                 return -ENOMEM;
 355         }
 356         DRM_INFO("doorbell mmio base: 0x%08X\n", (uint32_t)adev->doorbell.base);
 357         DRM_INFO("doorbell mmio size: %u\n", (unsigned)adev->doorbell.size);
 358
 359         return 0;
 360 }
 361
 362 /**
 363  * amdgpu_doorbell_fini - Tear down doorbell driver information.
 364  *
 365  * @adev: amdgpu_device pointer
 366  *
 367  * Tear down doorbell driver information (CIK)
 368  */
 369 static void amdgpu_doorbell_fini(struct amdgpu_device *adev)
 370 {
 371         iounmap(adev->doorbell.ptr);
 372         adev->doorbell.ptr = NULL;
 373 }
 374
 375 /**
 376  * amdgpu_doorbell_get_kfd_info - Report doorbell configuration required to
 377  *                                setup amdkfd
 378  *
 379  * @adev: amdgpu_device pointer
 380  * @aperture_base: output returning doorbell aperture base physical address
 381  * @aperture_size: output returning doorbell aperture size in bytes
 382  * @start_offset: output returning # of doorbell bytes reserved for amdgpu.
 383  *
 384  * amdgpu and amdkfd share the doorbell aperture. amdgpu sets it up,
 385  * takes doorbells required for its own rings and reports the setup to amdkfd.
 386  * amdgpu reserved doorbells are at the start of the doorbell aperture.
 387  */
 388 void amdgpu_doorbell_get_kfd_info(struct amdgpu_device *adev,
 389                                 phys_addr_t *aperture_base,
 390                                 size_t *aperture_size,
 391                                 size_t *start_offset)
 392 {
 393         /*
 394          * The first num_doorbells are used by amdgpu.
 395          * amdkfd takes whatever's left in the aperture.
 396          */
 397         if (adev->doorbell.size > adev->doorbell.num_doorbells * sizeof(u32)) {
 398                 *aperture_base = adev->doorbell.base;
 399                 *aperture_size = adev->doorbell.size;
 400                 *start_offset = adev->doorbell.num_doorbells * sizeof(u32);
 401         } else {
 402                 *aperture_base = 0;
 403                 *aperture_size = 0;
 404                 *start_offset = 0;
 405         }
 406 }
 407
 408 /*
 409  * amdgpu_wb_*()
 410  * Writeback is the the method by which the the GPU updates special pages
 411  * in memory with the status of certain GPU events (fences, ring pointers,
 412  * etc.).
 413  */
 414
 415 /**
 416  * amdgpu_wb_fini - Disable Writeback and free memory
 417  *
 418  * @adev: amdgpu_device pointer
 419  *
 420  * Disables Writeback and frees the Writeback memory (all asics).
 421  * Used at driver shutdown.
 422  */
 423 static void amdgpu_wb_fini(struct amdgpu_device *adev)
 424 {
 425         if (adev->wb.wb_obj) {
 426                 if (!amdgpu_bo_reserve(adev->wb.wb_obj, false)) {
 427                         amdgpu_bo_kunmap(adev->wb.wb_obj);
 428                         amdgpu_bo_unpin(adev->wb.wb_obj);
 429                         amdgpu_bo_unreserve(adev->wb.wb_obj);
 430                 }
 431                 amdgpu_bo_unref(&adev->wb.wb_obj);
 432                 adev->wb.wb = NULL;
 433                 adev->wb.wb_obj = NULL;
 434         }
 435 }
 436
 437 /**
 438  * amdgpu_wb_init- Init Writeback driver info and allocate memory
 439  *
 440  * @adev: amdgpu_device pointer
 441  *
 442  * Disables Writeback and frees the Writeback memory (all asics).
 443  * Used at driver startup.
 444  * Returns 0 on success or an -error on failure.
 445  */
 446 static int amdgpu_wb_init(struct amdgpu_device *adev)
 447 {
 448         int r;
 449
 450         if (adev->wb.wb_obj == NULL) {
 451                 r = amdgpu_bo_create(adev, AMDGPU_MAX_WB * 4, PAGE_SIZE, true,
 452                                      AMDGPU_GEM_DOMAIN_GTT, 0,  NULL, NULL,
 453                                      &adev->wb.wb_obj);
 454                 if (r) {
 455                         dev_warn(adev->dev, "(%d) create WB bo failed\n", r);
 456                         return r;
 457                 }
 458                 r = amdgpu_bo_reserve(adev->wb.wb_obj, false);
 459                 if (unlikely(r != 0)) {
 460                         amdgpu_wb_fini(adev);
 461                         return r;
 462                 }
 463                 r = amdgpu_bo_pin(adev->wb.wb_obj, AMDGPU_GEM_DOMAIN_GTT,
 464                                 &adev->wb.gpu_addr);
 465                 if (r) {
 466                         amdgpu_bo_unreserve(adev->wb.wb_obj);
 467                         dev_warn(adev->dev, "(%d) pin WB bo failed\n", r);
 468                         amdgpu_wb_fini(adev);
 469                         return r;
 470                 }
 471                 r = amdgpu_bo_kmap(adev->wb.wb_obj, (void **)&adev->wb.wb);
 472                 amdgpu_bo_unreserve(adev->wb.wb_obj);
 473                 if (r) {
 474                         dev_warn(adev->dev, "(%d) map WB bo failed\n", r);
 475                         amdgpu_wb_fini(adev);
 476                         return r;
 477                 }
 478
 479                 adev->wb.num_wb = AMDGPU_MAX_WB;
 480                 memset(&adev->wb.used, 0, sizeof(adev->wb.used));
 481
 482                 /* clear wb memory */
 483                 memset((char *)adev->wb.wb, 0, AMDGPU_GPU_PAGE_SIZE);
 484         }
 485
 486         return 0;
 487 }
 488
 489 /**
 490  * amdgpu_wb_get - Allocate a wb entry
 491  *
 492  * @adev: amdgpu_device pointer
 493  * @wb: wb index
 494  *
 495  * Allocate a wb slot for use by the driver (all asics).
 496  * Returns 0 on success or -EINVAL on failure.
 497  */
 498 int amdgpu_wb_get(struct amdgpu_device *adev, u32 *wb)
 499 {
 500         unsigned long offset = find_first_zero_bit(adev->wb.used, adev->wb.num_wb);
 501         if (offset < adev->wb.num_wb) {
 502                 __set_bit(offset, adev->wb.used);
 503                 *wb = offset;
 504                 return 0;
 505         } else {
 506                 return -EINVAL;
 507         }
 508 }
 509
 510 /**
 511  * amdgpu_wb_free - Free a wb entry
 512  *
 513  * @adev: amdgpu_device pointer
 514  * @wb: wb index
 515  *
 516  * Free a wb slot allocated for use by the driver (all asics)
 517  */
 518 void amdgpu_wb_free(struct amdgpu_device *adev, u32 wb)
 519 {
 520         if (wb < adev->wb.num_wb)
 521                 __clear_bit(wb, adev->wb.used);
 522 }
 523
 524 /**
 525  * amdgpu_vram_location - try to find VRAM location
 526  * @adev: amdgpu device structure holding all necessary informations
 527  * @mc: memory controller structure holding memory informations
 528  * @base: base address at which to put VRAM
 529  *
 530  * Function will place try to place VRAM at base address provided
 531  * as parameter (which is so far either PCI aperture address or
 532  * for IGP TOM base address).
 533  *
 534  * If there is not enough space to fit the unvisible VRAM in the 32bits
 535  * address space then we limit the VRAM size to the aperture.
 536  *
 537  * Note: We don't explicitly enforce VRAM start to be aligned on VRAM size,
 538  * this shouldn't be a problem as we are using the PCI aperture as a reference.
 539  * Otherwise this would be needed for rv280, all r3xx, and all r4xx, but
 540  * not IGP.
 541  *
 542  * Note: we use mc_vram_size as on some board we need to program the mc to
 543  * cover the whole aperture even if VRAM size is inferior to aperture size
 544  * Novell bug 204882 + along with lots of ubuntu ones
 545  *
 546  * Note: when limiting vram it's safe to overwritte real_vram_size because
 547  * we are not in case where real_vram_size is inferior to mc_vram_size (ie
 548  * note afected by bogus hw of Novell bug 204882 + along with lots of ubuntu
 549  * ones)
 550  *
 551  * Note: IGP TOM addr should be the same as the aperture addr, we don't
 552  * explicitly check for that thought.
 553  *
 554  * FIXME: when reducing VRAM size align new size on power of 2.
 555  */
 556 void amdgpu_vram_location(struct amdgpu_device *adev, struct amdgpu_mc *mc, u64 base)
 557 {
 558         uint64_t limit = (uint64_t)amdgpu_vram_limit << 20;
 559
 560         mc->vram_start = base;
 561         if (mc->mc_vram_size > (adev->mc.mc_mask - base + 1)) {
 562                 dev_warn(adev->dev, "limiting VRAM to PCI aperture size\n");
 563                 mc->real_vram_size = mc->aper_size;
 564                 mc->mc_vram_size = mc->aper_size;
 565         }
 566         mc->vram_end = mc->vram_start + mc->mc_vram_size - 1;
 567         if (limit && limit < mc->real_vram_size)
 568                 mc->real_vram_size = limit;
 569         dev_info(adev->dev, "VRAM: %lluM 0x%016llX - 0x%016llX (%lluM used)\n",
 570                         mc->mc_vram_size >> 20, mc->vram_start,
 571                         mc->vram_end, mc->real_vram_size >> 20);
 572 }
 573
 574 /**
 575  * amdgpu_gtt_location - try to find GTT location
 576  * @adev: amdgpu device structure holding all necessary informations
 577  * @mc: memory controller structure holding memory informations
 578  *
 579  * Function will place try to place GTT before or after VRAM.
 580  *
 581  * If GTT size is bigger than space left then we ajust GTT size.
 582  * Thus function will never fails.
 583  *
 584  * FIXME: when reducing GTT size align new size on power of 2.
 585  */
 586 void amdgpu_gtt_location(struct amdgpu_device *adev, struct amdgpu_mc *mc)
 587 {
 588         u64 size_af, size_bf;
 589
 590         size_af = ((adev->mc.mc_mask - mc->vram_end) + mc->gtt_base_align) & ~mc->gtt_base_align;
 591         size_bf = mc->vram_start & ~mc->gtt_base_align;
 592         if (size_bf > size_af) {
 593                 if (mc->gtt_size > size_bf) {
 594                         dev_warn(adev->dev, "limiting GTT\n");
 595                         mc->gtt_size = size_bf;
 596                 }
 597                 mc->gtt_start = (mc->vram_start & ~mc->gtt_base_align) - mc->gtt_size;
 598         } else {
 599                 if (mc->gtt_size > size_af) {
 600                         dev_warn(adev->dev, "limiting GTT\n");
 601                         mc->gtt_size = size_af;
 602                 }
 603                 mc->gtt_start = (mc->vram_end + 1 + mc->gtt_base_align) & ~mc->gtt_base_align;
 604         }
 605         mc->gtt_end = mc->gtt_start + mc->gtt_size - 1;
 606         dev_info(adev->dev, "GTT: %lluM 0x%016llX - 0x%016llX\n",
 607                         mc->gtt_size >> 20, mc->gtt_start, mc->gtt_end);
 608 }
 609
 610 /*
 611  * GPU helpers function.
 612  */
 613 /**
 614  * amdgpu_card_posted - check if the hw has already been initialized
 615  *
 616  * @adev: amdgpu_device pointer
 617  *
 618  * Check if the asic has been initialized (all asics).
 619  * Used at driver startup.
 620  * Returns true if initialized or false if not.
 621  */
 622 bool amdgpu_card_posted(struct amdgpu_device *adev)
 623 {
 624         uint32_t reg;
 625
 626         /* then check MEM_SIZE, in case the crtcs are off */
 627         reg = RREG32(mmCONFIG_MEMSIZE);
 628
 629         if (reg)
 630                 return true;
 631
 632         return false;
 633
 634 }
 635
 636 /**
 637  * amdgpu_boot_test_post_card - check and possibly initialize the hw
 638  *
 639  * @adev: amdgpu_device pointer
 640  *
 641  * Check if the asic is initialized and if not, attempt to initialize
 642  * it (all asics).
 643  * Returns true if initialized or false if not.
 644  */
 645 bool amdgpu_boot_test_post_card(struct amdgpu_device *adev)
 646 {
 647         if (amdgpu_card_posted(adev))
 648                 return true;
 649
 650         if (adev->bios) {
 651                 DRM_INFO("GPU not posted. posting now...\n");
 652                 if (adev->is_atom_bios)
 653                         amdgpu_atom_asic_init(adev->mode_info.atom_context);
 654                 return true;
 655         } else {
 656                 dev_err(adev->dev, "Card not posted and no BIOS - ignoring\n");
 657                 return false;
 658         }
 659 }
 660
 661 /**
 662  * amdgpu_dummy_page_init - init dummy page used by the driver
 663  *
 664  * @adev: amdgpu_device pointer
 665  *
 666  * Allocate the dummy page used by the driver (all asics).
 667  * This dummy page is used by the driver as a filler for gart entries
 668  * when pages are taken out of the GART
 669  * Returns 0 on sucess, -ENOMEM on failure.
 670  */
 671 int amdgpu_dummy_page_init(struct amdgpu_device *adev)
 672 {
 673         if (adev->dummy_page.page)
 674                 return 0;
 675         adev->dummy_page.page = alloc_page(GFP_DMA32 | GFP_KERNEL | __GFP_ZERO);
 676         if (adev->dummy_page.page == NULL)
 677                 return -ENOMEM;
 678         adev->dummy_page.addr = pci_map_page(adev->pdev, adev->dummy_page.page,
 679                                         0, PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
 680         if (pci_dma_mapping_error(adev->pdev, adev->dummy_page.addr)) {
 681                 dev_err(&adev->pdev->dev, "Failed to DMA MAP the dummy page\n");
 682                 __free_page(adev->dummy_page.page);
 683                 adev->dummy_page.page = NULL;
 684                 return -ENOMEM;
 685         }
 686         return 0;
 687 }
 688
 689 /**
 690  * amdgpu_dummy_page_fini - free dummy page used by the driver
 691  *
 692  * @adev: amdgpu_device pointer
 693  *
 694  * Frees the dummy page used by the driver (all asics).
 695  */
 696 void amdgpu_dummy_page_fini(struct amdgpu_device *adev)
 697 {
 698         if (adev->dummy_page.page == NULL)
 699                 return;
 700         pci_unmap_page(adev->pdev, adev->dummy_page.addr,
 701                         PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
 702         __free_page(adev->dummy_page.page);
 703         adev->dummy_page.page = NULL;
 704 }
 705
 706
 707 /* ATOM accessor methods */
 708 /*
 709  * ATOM is an interpreted byte code stored in tables in the vbios.  The
 710  * driver registers callbacks to access registers and the interpreter
 711  * in the driver parses the tables and executes then to program specific
 712  * actions (set display modes, asic init, etc.).  See amdgpu_atombios.c,
 713  * atombios.h, and atom.c
 714  */
 715
 716 /**
 717  * cail_pll_read - read PLL register
 718  *
 719  * @info: atom card_info pointer
 720  * @reg: PLL register offset
 721  *
 722  * Provides a PLL register accessor for the atom interpreter (r4xx+).
 723  * Returns the value of the PLL register.
 724  */
 725 static uint32_t cail_pll_read(struct card_info *info, uint32_t reg)
 726 {
 727         return 0;
 728 }
 729
 730 /**
 731  * cail_pll_write - write PLL register
 732  *
 733  * @info: atom card_info pointer
 734  * @reg: PLL register offset
 735  * @val: value to write to the pll register
 736  *
 737  * Provides a PLL register accessor for the atom interpreter (r4xx+).
 738  */
 739 static void cail_pll_write(struct card_info *info, uint32_t reg, uint32_t val)
 740 {
 741
 742 }
 743
 744 /**
 745  * cail_mc_read - read MC (Memory Controller) register
 746  *
 747  * @info: atom card_info pointer
 748  * @reg: MC register offset
 749  *
 750  * Provides an MC register accessor for the atom interpreter (r4xx+).
 751  * Returns the value of the MC register.
 752  */
 753 static uint32_t cail_mc_read(struct card_info *info, uint32_t reg)
 754 {
 755         return 0;
 756 }
 757
 758 /**
 759  * cail_mc_write - write MC (Memory Controller) register
 760  *
 761  * @info: atom card_info pointer
 762  * @reg: MC register offset
 763  * @val: value to write to the pll register
 764  *
 765  * Provides a MC register accessor for the atom interpreter (r4xx+).
 766  */
 767 static void cail_mc_write(struct card_info *info, uint32_t reg, uint32_t val)
 768 {
 769
 770 }
 771
 772 /**
 773  * cail_reg_write - write MMIO register
 774  *
 775  * @info: atom card_info pointer
 776  * @reg: MMIO register offset
 777  * @val: value to write to the pll register
 778  *
 779  * Provides a MMIO register accessor for the atom interpreter (r4xx+).
 780  */
 781 static void cail_reg_write(struct card_info *info, uint32_t reg, uint32_t val)
 782 {
 783         struct amdgpu_device *adev = info->dev->dev_private;
 784
 785         WREG32(reg, val);
 786 }
 787
 788 /**
 789  * cail_reg_read - read MMIO register
 790  *
 791  * @info: atom card_info pointer
 792  * @reg: MMIO register offset
 793  *
 794  * Provides an MMIO register accessor for the atom interpreter (r4xx+).
 795  * Returns the value of the MMIO register.
 796  */
 797 static uint32_t cail_reg_read(struct card_info *info, uint32_t reg)
 798 {
 799         struct amdgpu_device *adev = info->dev->dev_private;
 800         uint32_t r;
 801
 802         r = RREG32(reg);
 803         return r;
 804 }
 805
 806 /**
 807  * cail_ioreg_write - write IO register
 808  *
 809  * @info: atom card_info pointer
 810  * @reg: IO register offset
 811  * @val: value to write to the pll register
 812  *
 813  * Provides a IO register accessor for the atom interpreter (r4xx+).
 814  */
 815 static void cail_ioreg_write(struct card_info *info, uint32_t reg, uint32_t val)
 816 {
 817         struct amdgpu_device *adev = info->dev->dev_private;
 818
 819         WREG32_IO(reg, val);
 820 }
 821
 822 /**
 823  * cail_ioreg_read - read IO register
 824  *
 825  * @info: atom card_info pointer
 826  * @reg: IO register offset
 827  *
 828  * Provides an IO register accessor for the atom interpreter (r4xx+).
 829  * Returns the value of the IO register.
 830  */
 831 static uint32_t cail_ioreg_read(struct card_info *info, uint32_t reg)
 832 {
 833         struct amdgpu_device *adev = info->dev->dev_private;
 834         uint32_t r;
 835
 836         r = RREG32_IO(reg);
 837         return r;
 838 }
 839
 840 /**
 841  * amdgpu_atombios_fini - free the driver info and callbacks for atombios
 842  *
 843  * @adev: amdgpu_device pointer
 844  *
 845  * Frees the driver info and register access callbacks for the ATOM
 846  * interpreter (r4xx+).
 847  * Called at driver shutdown.
 848  */
 849 static void amdgpu_atombios_fini(struct amdgpu_device *adev)
 850 {
 851         if (adev->mode_info.atom_context)
 852                 kfree(adev->mode_info.atom_context->scratch);
 853         kfree(adev->mode_info.atom_context);
 854         adev->mode_info.atom_context = NULL;
 855         kfree(adev->mode_info.atom_card_info);
 856         adev->mode_info.atom_card_info = NULL;
 857 }
 858
 859 /**
 860  * amdgpu_atombios_init - init the driver info and callbacks for atombios
 861  *
 862  * @adev: amdgpu_device pointer
 863  *
 864  * Initializes the driver info and register access callbacks for the
 865  * ATOM interpreter (r4xx+).
 866  * Returns 0 on sucess, -ENOMEM on failure.
 867  * Called at driver startup.
 868  */
 869 static int amdgpu_atombios_init(struct amdgpu_device *adev)
 870 {
 871         struct card_info *atom_card_info =
 872             kzalloc(sizeof(struct card_info), GFP_KERNEL);
 873
 874         if (!atom_card_info)
 875                 return -ENOMEM;
 876
 877         adev->mode_info.atom_card_info = atom_card_info;
 878         atom_card_info->dev = adev->ddev;
 879         atom_card_info->reg_read = cail_reg_read;
 880         atom_card_info->reg_write = cail_reg_write;
 881         /* needed for iio ops */
 882         if (adev->rio_mem) {
 883                 atom_card_info->ioreg_read = cail_ioreg_read;
 884                 atom_card_info->ioreg_write = cail_ioreg_write;
 885         } else {
 886                 DRM_ERROR("Unable to find PCI I/O BAR; using MMIO for ATOM IIO\n");
 887                 atom_card_info->ioreg_read = cail_reg_read;
 888                 atom_card_info->ioreg_write = cail_reg_write;
 889         }
 890         atom_card_info->mc_read = cail_mc_read;
 891         atom_card_info->mc_write = cail_mc_write;
 892         atom_card_info->pll_read = cail_pll_read;
 893         atom_card_info->pll_write = cail_pll_write;
 894
 895         adev->mode_info.atom_context = amdgpu_atom_parse(atom_card_info, adev->bios);
 896         if (!adev->mode_info.atom_context) {
 897                 amdgpu_atombios_fini(adev);
 898                 return -ENOMEM;
 899         }
 900
 901         mutex_init(&adev->mode_info.atom_context->mutex);
 902         amdgpu_atombios_scratch_regs_init(adev);
 903         amdgpu_atom_allocate_fb_scratch(adev->mode_info.atom_context);
 904         return 0;
 905 }
 906
 907 /* if we get transitioned to only one device, take VGA back */
 908 /**
 909  * amdgpu_vga_set_decode - enable/disable vga decode
 910  *
 911  * @cookie: amdgpu_device pointer
 912  * @state: enable/disable vga decode
 913  *
 914  * Enable/disable vga decode (all asics).
 915  * Returns VGA resource flags.
 916  */
 917 static unsigned int amdgpu_vga_set_decode(void *cookie, bool state)
 918 {
 919         struct amdgpu_device *adev = cookie;
 920         amdgpu_asic_set_vga_state(adev, state);
 921         if (state)
 922                 return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM |
 923                        VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
 924         else
 925                 return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
 926 }
 927
 928 /**
 929  * amdgpu_check_pot_argument - check that argument is a power of two
 930  *
 931  * @arg: value to check
 932  *
 933  * Validates that a certain argument is a power of two (all asics).
 934  * Returns true if argument is valid.
 935  */
 936 static bool amdgpu_check_pot_argument(int arg)
 937 {
 938         return (arg & (arg - 1)) == 0;
 939 }
 940
 941 /**
 942  * amdgpu_check_arguments - validate module params
 943  *
 944  * @adev: amdgpu_device pointer
 945  *
 946  * Validates certain module parameters and updates
 947  * the associated values used by the driver (all asics).
 948  */
 949 static void amdgpu_check_arguments(struct amdgpu_device *adev)
 950 {
 951         /* vramlimit must be a power of two */
 952         if (!amdgpu_check_pot_argument(amdgpu_vram_limit)) {
 953                 dev_warn(adev->dev, "vram limit (%d) must be a power of 2\n",
 954                                 amdgpu_vram_limit);
 955                 amdgpu_vram_limit = 0;
 956         }
 957
 958         if (amdgpu_gart_size != -1) {
 959                 /* gtt size must be power of two and greater or equal to 32M */
 960                 if (amdgpu_gart_size < 32) {
 961                         dev_warn(adev->dev, "gart size (%d) too small\n",
 962                                  amdgpu_gart_size);
 963                         amdgpu_gart_size = -1;
 964                 } else if (!amdgpu_check_pot_argument(amdgpu_gart_size)) {
 965                         dev_warn(adev->dev, "gart size (%d) must be a power of 2\n",
 966                                  amdgpu_gart_size);
 967                         amdgpu_gart_size = -1;
 968                 }
 969         }
 970
 971         if (!amdgpu_check_pot_argument(amdgpu_vm_size)) {
 972                 dev_warn(adev->dev, "VM size (%d) must be a power of 2\n",
 973                          amdgpu_vm_size);
 974                 amdgpu_vm_size = 8;
 975         }
 976
 977         if (amdgpu_vm_size < 1) {
 978                 dev_warn(adev->dev, "VM size (%d) too small, min is 1GB\n",
 979                          amdgpu_vm_size);
 980                 amdgpu_vm_size = 8;
 981         }
 982
 983         /*
 984          * Max GPUVM size for Cayman, SI and CI are 40 bits.
 985          */
 986         if (amdgpu_vm_size > 1024) {
 987                 dev_warn(adev->dev, "VM size (%d) too large, max is 1TB\n",
 988                          amdgpu_vm_size);
 989                 amdgpu_vm_size = 8;
 990         }
 991
 992         /* defines number of bits in page table versus page directory,
 993          * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
 994          * page table and the remaining bits are in the page directory */
 995         if (amdgpu_vm_block_size == -1) {
 996
 997                 /* Total bits covered by PD + PTs */
 998                 unsigned bits = ilog2(amdgpu_vm_size) + 18;
 999
1000                 /* Make sure the PD is 4K in size up to 8GB address space.
1001                    Above that split equal between PD and PTs */
1002                 if (amdgpu_vm_size <= 8)
1003                         amdgpu_vm_block_size = bits - 9;
1004                 else
1005                         amdgpu_vm_block_size = (bits + 3) / 2;
1006
1007         } else if (amdgpu_vm_block_size < 9) {
1008                 dev_warn(adev->dev, "VM page table size (%d) too small\n",
1009                          amdgpu_vm_block_size);
1010                 amdgpu_vm_block_size = 9;
1011         }
1012
1013         if (amdgpu_vm_block_size > 24 ||
1014             (amdgpu_vm_size * 1024) < (1ull << amdgpu_vm_block_size)) {
1015                 dev_warn(adev->dev, "VM page table size (%d) too large\n",
1016                          amdgpu_vm_block_size);
1017                 amdgpu_vm_block_size = 9;
1018         }
1019 }
1020
1021 /**
1022  * amdgpu_switcheroo_set_state - set switcheroo state
1023  *
1024  * @pdev: pci dev pointer
1025  * @state: vga switcheroo state
1026  *
1027  * Callback for the switcheroo driver.  Suspends or resumes the
1028  * the asics before or after it is powered up using ACPI methods.
1029  */
1030 static void amdgpu_switcheroo_set_state(struct pci_dev *pdev, enum vga_switcheroo_state state)
1031 {
1032         struct drm_device *dev = pci_get_drvdata(pdev);
1033
1034         if (amdgpu_device_is_px(dev) && state == VGA_SWITCHEROO_OFF)
1035                 return;
1036
1037         if (state == VGA_SWITCHEROO_ON) {
1038                 unsigned d3_delay = dev->pdev->d3_delay;
1039
1040                 printk(KERN_INFO "amdgpu: switched on\n");
1041                 /* don't suspend or resume card normally */
1042                 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1043
1044                 amdgpu_resume_kms(dev, true, true);
1045
1046                 dev->pdev->d3_delay = d3_delay;
1047
1048                 dev->switch_power_state = DRM_SWITCH_POWER_ON;
1049                 drm_kms_helper_poll_enable(dev);
1050         } else {
1051                 printk(KERN_INFO "amdgpu: switched off\n");
1052                 drm_kms_helper_poll_disable(dev);
1053                 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1054                 amdgpu_suspend_kms(dev, true, true);
1055                 dev->switch_power_state = DRM_SWITCH_POWER_OFF;
1056         }
1057 }
1058
1059 /**
1060  * amdgpu_switcheroo_can_switch - see if switcheroo state can change
1061  *
1062  * @pdev: pci dev pointer
1063  *
1064  * Callback for the switcheroo driver.  Check of the switcheroo
1065  * state can be changed.
1066  * Returns true if the state can be changed, false if not.
1067  */
1068 static bool amdgpu_switcheroo_can_switch(struct pci_dev *pdev)
1069 {
1070         struct drm_device *dev = pci_get_drvdata(pdev);
1071
1072         /*
1073         * FIXME: open_count is protected by drm_global_mutex but that would lead to
1074         * locking inversion with the driver load path. And the access here is
1075         * completely racy anyway. So don't bother with locking for now.
1076         */
1077         return dev->open_count == 0;
1078 }
1079
1080 static const struct vga_switcheroo_client_ops amdgpu_switcheroo_ops = {
1081         .set_gpu_state = amdgpu_switcheroo_set_state,
1082         .reprobe = NULL,
1083         .can_switch = amdgpu_switcheroo_can_switch,
1084 };
1085
1086 int amdgpu_set_clockgating_state(struct amdgpu_device *adev,
1087                                   enum amd_ip_block_type block_type,
1088                                   enum amd_clockgating_state state)
1089 {
1090         int i, r = 0;
1091
1092         for (i = 0; i < adev->num_ip_blocks; i++) {
1093                 if (adev->ip_blocks[i].type == block_type) {
1094                         r = adev->ip_blocks[i].funcs->set_clockgating_state((void *)adev,
1095                                                                             state);
1096                         if (r)
1097                                 return r;
1098                 }
1099         }
1100         return r;
1101 }
1102
1103 int amdgpu_set_powergating_state(struct amdgpu_device *adev,
1104                                   enum amd_ip_block_type block_type,
1105                                   enum amd_powergating_state state)
1106 {
1107         int i, r = 0;
1108
1109         for (i = 0; i < adev->num_ip_blocks; i++) {
1110                 if (adev->ip_blocks[i].type == block_type) {
1111                         r = adev->ip_blocks[i].funcs->set_powergating_state((void *)adev,
1112                                                                             state);
1113                         if (r)
1114                                 return r;
1115                 }
1116         }
1117         return r;
1118 }
1119
1120 const struct amdgpu_ip_block_version * amdgpu_get_ip_block(
1121                                         struct amdgpu_device *adev,
1122                                         enum amd_ip_block_type type)
1123 {
1124         int i;
1125
1126         for (i = 0; i < adev->num_ip_blocks; i++)
1127                 if (adev->ip_blocks[i].type == type)
1128                         return &adev->ip_blocks[i];
1129
1130         return NULL;
1131 }
1132
1133 /**
1134  * amdgpu_ip_block_version_cmp
1135  *
1136  * @adev: amdgpu_device pointer
1137  * @type: enum amd_ip_block_type
1138  * @major: major version
1139  * @minor: minor version
1140  *
1141  * return 0 if equal or greater
1142  * return 1 if smaller or the ip_block doesn't exist
1143  */
1144 int amdgpu_ip_block_version_cmp(struct amdgpu_device *adev,
1145                                 enum amd_ip_block_type type,
1146                                 u32 major, u32 minor)
1147 {
1148         const struct amdgpu_ip_block_version *ip_block;
1149         ip_block = amdgpu_get_ip_block(adev, type);
1150
1151         if (ip_block && ((ip_block->major > major) ||
1152                         ((ip_block->major == major) &&
1153                         (ip_block->minor >= minor))))
1154                 return 0;
1155
1156         return 1;
1157 }
1158
1159 static int amdgpu_early_init(struct amdgpu_device *adev)
1160 {
1161         int i, r;
1162
1163         switch (adev->asic_type) {
1164         case CHIP_TOPAZ:
1165         case CHIP_TONGA:
1166         case CHIP_FIJI:
1167         case CHIP_CARRIZO:
1168                 if (adev->asic_type == CHIP_CARRIZO)
1169                         adev->family = AMDGPU_FAMILY_CZ;
1170                 else
1171                         adev->family = AMDGPU_FAMILY_VI;
1172
1173                 r = vi_set_ip_blocks(adev);
1174                 if (r)
1175                         return r;
1176                 break;
1177 #ifdef CONFIG_DRM_AMDGPU_CIK
1178         case CHIP_BONAIRE:
1179         case CHIP_HAWAII:
1180         case CHIP_KAVERI:
1181         case CHIP_KABINI:
1182         case CHIP_MULLINS:
1183                 if ((adev->asic_type == CHIP_BONAIRE) || (adev->asic_type == CHIP_HAWAII))
1184                         adev->family = AMDGPU_FAMILY_CI;
1185                 else
1186                         adev->family = AMDGPU_FAMILY_KV;
1187
1188                 r = cik_set_ip_blocks(adev);
1189                 if (r)
1190                         return r;
1191                 break;
1192 #endif
1193         default:
1194                 /* FIXME: not supported yet */
1195                 return -EINVAL;
1196         }
1197
1198         adev->ip_block_status = kcalloc(adev->num_ip_blocks,
1199                                         sizeof(struct amdgpu_ip_block_status), GFP_KERNEL);
1200         if (adev->ip_block_status == NULL)
1201                 return -ENOMEM;
1202
1203         if (adev->ip_blocks == NULL) {
1204                 DRM_ERROR("No IP blocks found!\n");
1205                 return r;
1206         }
1207
1208         for (i = 0; i < adev->num_ip_blocks; i++) {
1209                 if ((amdgpu_ip_block_mask & (1 << i)) == 0) {
1210                         DRM_ERROR("disabled ip block: %d\n", i);
1211                         adev->ip_block_status[i].valid = false;
1212                 } else {
1213                         if (adev->ip_blocks[i].funcs->early_init) {
1214                                 r = adev->ip_blocks[i].funcs->early_init((void *)adev);
1215                                 if (r == -ENOENT)
1216                                         adev->ip_block_status[i].valid = false;
1217                                 else if (r)
1218                                         return r;
1219                                 else
1220                                         adev->ip_block_status[i].valid = true;
1221                         } else {
1222                                 adev->ip_block_status[i].valid = true;
1223                         }
1224                 }
1225         }
1226
1227         return 0;
1228 }
1229
1230 static int amdgpu_init(struct amdgpu_device *adev)
1231 {
1232         int i, r;
1233
1234         for (i = 0; i < adev->num_ip_blocks; i++) {
1235                 if (!adev->ip_block_status[i].valid)
1236                         continue;
1237                 r = adev->ip_blocks[i].funcs->sw_init((void *)adev);
1238                 if (r)
1239                         return r;
1240                 adev->ip_block_status[i].sw = true;
1241                 /* need to do gmc hw init early so we can allocate gpu mem */
1242                 if (adev->ip_blocks[i].type == AMD_IP_BLOCK_TYPE_GMC) {
1243                         r = amdgpu_vram_scratch_init(adev);
1244                         if (r)
1245                                 return r;
1246                         r = adev->ip_blocks[i].funcs->hw_init((void *)adev);
1247                         if (r)
1248                                 return r;
1249                         r = amdgpu_wb_init(adev);
1250                         if (r)
1251                                 return r;
1252                         adev->ip_block_status[i].hw = true;
1253                 }
1254         }
1255
1256         for (i = 0; i < adev->num_ip_blocks; i++) {
1257                 if (!adev->ip_block_status[i].sw)
1258                         continue;
1259                 /* gmc hw init is done early */
1260                 if (adev->ip_blocks[i].type == AMD_IP_BLOCK_TYPE_GMC)
1261                         continue;
1262                 r = adev->ip_blocks[i].funcs->hw_init((void *)adev);
1263                 if (r)
1264                         return r;
1265                 adev->ip_block_status[i].hw = true;
1266         }
1267
1268         return 0;
1269 }
1270
1271 static int amdgpu_late_init(struct amdgpu_device *adev)
1272 {
1273         int i = 0, r;
1274
1275         for (i = 0; i < adev->num_ip_blocks; i++) {
1276                 if (!adev->ip_block_status[i].valid)
1277                         continue;
1278                 /* enable clockgating to save power */
1279                 r = adev->ip_blocks[i].funcs->set_clockgating_state((void *)adev,
1280                                                                     AMD_CG_STATE_GATE);
1281                 if (r)
1282                         return r;
1283                 if (adev->ip_blocks[i].funcs->late_init) {
1284                         r = adev->ip_blocks[i].funcs->late_init((void *)adev);
1285                         if (r)
1286                                 return r;
1287                 }
1288         }
1289
1290         return 0;
1291 }
1292
1293 static int amdgpu_fini(struct amdgpu_device *adev)
1294 {
1295         int i, r;
1296
1297         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
1298                 if (!adev->ip_block_status[i].hw)
1299                         continue;
1300                 if (adev->ip_blocks[i].type == AMD_IP_BLOCK_TYPE_GMC) {
1301                         amdgpu_wb_fini(adev);
1302                         amdgpu_vram_scratch_fini(adev);
1303                 }
1304                 /* ungate blocks before hw fini so that we can shutdown the blocks safely */
1305                 r = adev->ip_blocks[i].funcs->set_clockgating_state((void *)adev,
1306                                                                     AMD_CG_STATE_UNGATE);
1307                 if (r)
1308                         return r;
1309                 r = adev->ip_blocks[i].funcs->hw_fini((void *)adev);
1310                 /* XXX handle errors */
1311                 adev->ip_block_status[i].hw = false;
1312         }
1313
1314         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
1315                 if (!adev->ip_block_status[i].sw)
1316                         continue;
1317                 r = adev->ip_blocks[i].funcs->sw_fini((void *)adev);
1318                 /* XXX handle errors */
1319                 adev->ip_block_status[i].sw = false;
1320                 adev->ip_block_status[i].valid = false;
1321         }
1322
1323         return 0;
1324 }
1325
1326 static int amdgpu_suspend(struct amdgpu_device *adev)
1327 {
1328         int i, r;
1329
1330         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
1331                 if (!adev->ip_block_status[i].valid)
1332                         continue;
1333                 /* ungate blocks so that suspend can properly shut them down */
1334                 r = adev->ip_blocks[i].funcs->set_clockgating_state((void *)adev,
1335                                                                     AMD_CG_STATE_UNGATE);
1336                 /* XXX handle errors */
1337                 r = adev->ip_blocks[i].funcs->suspend(adev);
1338                 /* XXX handle errors */
1339         }
1340
1341         return 0;
1342 }
1343
1344 static int amdgpu_resume(struct amdgpu_device *adev)
1345 {
1346         int i, r;
1347
1348         for (i = 0; i < adev->num_ip_blocks; i++) {
1349                 if (!adev->ip_block_status[i].valid)
1350                         continue;
1351                 r = adev->ip_blocks[i].funcs->resume(adev);
1352                 if (r)
1353                         return r;
1354         }
1355
1356         return 0;
1357 }
1358
1359 /**
1360  * amdgpu_device_init - initialize the driver
1361  *
1362  * @adev: amdgpu_device pointer
1363  * @pdev: drm dev pointer
1364  * @pdev: pci dev pointer
1365  * @flags: driver flags
1366  *
1367  * Initializes the driver info and hw (all asics).
1368  * Returns 0 for success or an error on failure.
1369  * Called at driver startup.
1370  */
1371 int amdgpu_device_init(struct amdgpu_device *adev,
1372                        struct drm_device *ddev,
1373                        struct pci_dev *pdev,
1374                        uint32_t flags)
1375 {
1376         int r, i;
1377         bool runtime = false;
1378
1379         adev->shutdown = false;
1380         adev->dev = &pdev->dev;
1381         adev->ddev = ddev;
1382         adev->pdev = pdev;
1383         adev->flags = flags;
1384         adev->asic_type = flags & AMD_ASIC_MASK;
1385         adev->is_atom_bios = false;
1386         adev->usec_timeout = AMDGPU_MAX_USEC_TIMEOUT;
1387         adev->mc.gtt_size = 512 * 1024 * 1024;
1388         adev->accel_working = false;
1389         adev->num_rings = 0;
1390         adev->mman.buffer_funcs = NULL;
1391         adev->mman.buffer_funcs_ring = NULL;
1392         adev->vm_manager.vm_pte_funcs = NULL;
1393         adev->vm_manager.vm_pte_funcs_ring = NULL;
1394         adev->gart.gart_funcs = NULL;
1395         adev->fence_context = fence_context_alloc(AMDGPU_MAX_RINGS);
1396
1397         adev->smc_rreg = &amdgpu_invalid_rreg;
1398         adev->smc_wreg = &amdgpu_invalid_wreg;
1399         adev->pcie_rreg = &amdgpu_invalid_rreg;
1400         adev->pcie_wreg = &amdgpu_invalid_wreg;
1401         adev->uvd_ctx_rreg = &amdgpu_invalid_rreg;
1402         adev->uvd_ctx_wreg = &amdgpu_invalid_wreg;
1403         adev->didt_rreg = &amdgpu_invalid_rreg;
1404         adev->didt_wreg = &amdgpu_invalid_wreg;
1405         adev->audio_endpt_rreg = &amdgpu_block_invalid_rreg;
1406         adev->audio_endpt_wreg = &amdgpu_block_invalid_wreg;
1407
1408         DRM_INFO("initializing kernel modesetting (%s 0x%04X:0x%04X 0x%04X:0x%04X 0x%02X).\n",
1409                  amdgpu_asic_name[adev->asic_type], pdev->vendor, pdev->device,
1410                  pdev->subsystem_vendor, pdev->subsystem_device, pdev->revision);
1411
1412         /* mutex initialization are all done here so we
1413          * can recall function without having locking issues */
1414         mutex_init(&adev->ring_lock);
1415         atomic_set(&adev->irq.ih.lock, 0);
1416         mutex_init(&adev->gem.mutex);
1417         mutex_init(&adev->pm.mutex);
1418         mutex_init(&adev->gfx.gpu_clock_mutex);
1419         mutex_init(&adev->srbm_mutex);
1420         mutex_init(&adev->grbm_idx_mutex);
1421         init_rwsem(&adev->exclusive_lock);
1422         mutex_init(&adev->mn_lock);
1423         hash_init(adev->mn_hash);
1424
1425         amdgpu_check_arguments(adev);
1426
1427         /* Registers mapping */
1428         /* TODO: block userspace mapping of io register */
1429         spin_lock_init(&adev->mmio_idx_lock);
1430         spin_lock_init(&adev->smc_idx_lock);
1431         spin_lock_init(&adev->pcie_idx_lock);
1432         spin_lock_init(&adev->uvd_ctx_idx_lock);
1433         spin_lock_init(&adev->didt_idx_lock);
1434         spin_lock_init(&adev->audio_endpt_idx_lock);
1435
1436         adev->rmmio_base = pci_resource_start(adev->pdev, 5);
1437         adev->rmmio_size = pci_resource_len(adev->pdev, 5);
1438         adev->rmmio = ioremap(adev->rmmio_base, adev->rmmio_size);
1439         if (adev->rmmio == NULL) {
1440                 return -ENOMEM;
1441         }
1442         DRM_INFO("register mmio base: 0x%08X\n", (uint32_t)adev->rmmio_base);
1443         DRM_INFO("register mmio size: %u\n", (unsigned)adev->rmmio_size);
1444
1445         /* doorbell bar mapping */
1446         amdgpu_doorbell_init(adev);
1447
1448         /* io port mapping */
1449         for (i = 0; i < DEVICE_COUNT_RESOURCE; i++) {
1450                 if (pci_resource_flags(adev->pdev, i) & IORESOURCE_IO) {
1451                         adev->rio_mem_size = pci_resource_len(adev->pdev, i);
1452                         adev->rio_mem = pci_iomap(adev->pdev, i, adev->rio_mem_size);
1453                         break;
1454                 }
1455         }
1456         if (adev->rio_mem == NULL)
1457                 DRM_ERROR("Unable to find PCI I/O BAR\n");
1458
1459         /* early init functions */
1460         r = amdgpu_early_init(adev);
1461         if (r)
1462                 return r;
1463
1464         /* if we have > 1 VGA cards, then disable the amdgpu VGA resources */
1465         /* this will fail for cards that aren't VGA class devices, just
1466          * ignore it */
1467         vga_client_register(adev->pdev, adev, NULL, amdgpu_vga_set_decode);
1468
1469         if (amdgpu_runtime_pm == 1)
1470                 runtime = true;
1471         if (amdgpu_device_is_px(ddev))
1472                 runtime = true;
1473         vga_switcheroo_register_client(adev->pdev, &amdgpu_switcheroo_ops, runtime);
1474         if (runtime)
1475                 vga_switcheroo_init_domain_pm_ops(adev->dev, &adev->vga_pm_domain);
1476
1477         /* Read BIOS */
1478         if (!amdgpu_get_bios(adev))
1479                 return -EINVAL;
1480         /* Must be an ATOMBIOS */
1481         if (!adev->is_atom_bios) {
1482                 dev_err(adev->dev, "Expecting atombios for GPU\n");
1483                 return -EINVAL;
1484         }
1485         r = amdgpu_atombios_init(adev);
1486         if (r)
1487                 return r;
1488
1489         /* Post card if necessary */
1490         if (!amdgpu_card_posted(adev)) {
1491                 if (!adev->bios) {
1492                         dev_err(adev->dev, "Card not posted and no BIOS - ignoring\n");
1493                         return -EINVAL;
1494                 }
1495                 DRM_INFO("GPU not posted. posting now...\n");
1496                 amdgpu_atom_asic_init(adev->mode_info.atom_context);
1497         }
1498
1499         /* Initialize clocks */
1500         r = amdgpu_atombios_get_clock_info(adev);
1501         if (r)
1502                 return r;
1503         /* init i2c buses */
1504         amdgpu_atombios_i2c_init(adev);
1505
1506         /* Fence driver */
1507         r = amdgpu_fence_driver_init(adev);
1508         if (r)
1509                 return r;
1510
1511         /* init the mode config */
1512         drm_mode_config_init(adev->ddev);
1513
1514         r = amdgpu_init(adev);
1515         if (r) {
1516                 amdgpu_fini(adev);
1517                 return r;
1518         }
1519
1520         adev->accel_working = true;
1521
1522         amdgpu_fbdev_init(adev);
1523
1524         r = amdgpu_ib_pool_init(adev);
1525         if (r) {
1526                 dev_err(adev->dev, "IB initialization failed (%d).\n", r);
1527                 return r;
1528         }
1529
1530         r = amdgpu_ctx_init(adev, true, &adev->kernel_ctx);
1531         if (r) {
1532                 dev_err(adev->dev, "failed to create kernel context (%d).\n", r);
1533                 return r;
1534         }
1535         r = amdgpu_ib_ring_tests(adev);
1536         if (r)
1537                 DRM_ERROR("ib ring test failed (%d).\n", r);
1538
1539         r = amdgpu_gem_debugfs_init(adev);
1540         if (r) {
1541                 DRM_ERROR("registering gem debugfs failed (%d).\n", r);
1542         }
1543
1544         r = amdgpu_debugfs_regs_init(adev);
1545         if (r) {
1546                 DRM_ERROR("registering register debugfs failed (%d).\n", r);
1547         }
1548
1549         if ((amdgpu_testing & 1)) {
1550                 if (adev->accel_working)
1551                         amdgpu_test_moves(adev);
1552                 else
1553                         DRM_INFO("amdgpu: acceleration disabled, skipping move tests\n");
1554         }
1555         if ((amdgpu_testing & 2)) {
1556                 if (adev->accel_working)
1557                         amdgpu_test_syncing(adev);
1558                 else
1559                         DRM_INFO("amdgpu: acceleration disabled, skipping sync tests\n");
1560         }
1561         if (amdgpu_benchmarking) {
1562                 if (adev->accel_working)
1563                         amdgpu_benchmark(adev, amdgpu_benchmarking);
1564                 else
1565                         DRM_INFO("amdgpu: acceleration disabled, skipping benchmarks\n");
1566         }
1567
1568         /* enable clockgating, etc. after ib tests, etc. since some blocks require
1569          * explicit gating rather than handling it automatically.
1570          */
1571         r = amdgpu_late_init(adev);
1572         if (r)
1573                 return r;
1574
1575         return 0;
1576 }
1577
1578 static void amdgpu_debugfs_remove_files(struct amdgpu_device *adev);
1579
1580 /**
1581  * amdgpu_device_fini - tear down the driver
1582  *
1583  * @adev: amdgpu_device pointer
1584  *
1585  * Tear down the driver info (all asics).
1586  * Called at driver shutdown.
1587  */
1588 void amdgpu_device_fini(struct amdgpu_device *adev)
1589 {
1590         int r;
1591
1592         DRM_INFO("amdgpu: finishing device.\n");
1593         adev->shutdown = true;
1594         /* evict vram memory */
1595         amdgpu_bo_evict_vram(adev);
1596         amdgpu_ctx_fini(&adev->kernel_ctx);
1597         amdgpu_ib_pool_fini(adev);
1598         amdgpu_fence_driver_fini(adev);
1599         amdgpu_fbdev_fini(adev);
1600         r = amdgpu_fini(adev);
1601         kfree(adev->ip_block_status);
1602         adev->ip_block_status = NULL;
1603         adev->accel_working = false;
1604         /* free i2c buses */
1605         amdgpu_i2c_fini(adev);
1606         amdgpu_atombios_fini(adev);
1607         kfree(adev->bios);
1608         adev->bios = NULL;
1609         vga_switcheroo_unregister_client(adev->pdev);
1610         vga_client_register(adev->pdev, NULL, NULL, NULL);
1611         if (adev->rio_mem)
1612                 pci_iounmap(adev->pdev, adev->rio_mem);
1613         adev->rio_mem = NULL;
1614         iounmap(adev->rmmio);
1615         adev->rmmio = NULL;
1616         amdgpu_doorbell_fini(adev);
1617         amdgpu_debugfs_regs_cleanup(adev);
1618         amdgpu_debugfs_remove_files(adev);
1619 }
1620
1621
1622 /*
1623  * Suspend & resume.
1624  */
1625 /**
1626  * amdgpu_suspend_kms - initiate device suspend
1627  *
1628  * @pdev: drm dev pointer
1629  * @state: suspend state
1630  *
1631  * Puts the hw in the suspend state (all asics).
1632  * Returns 0 for success or an error on failure.
1633  * Called at driver suspend.
1634  */
1635 int amdgpu_suspend_kms(struct drm_device *dev, bool suspend, bool fbcon)
1636 {
1637         struct amdgpu_device *adev;
1638         struct drm_crtc *crtc;
1639         struct drm_connector *connector;
1640         int r;
1641
1642         if (dev == NULL || dev->dev_private == NULL) {
1643                 return -ENODEV;
1644         }
1645
1646         adev = dev->dev_private;
1647
1648         if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
1649                 return 0;
1650
1651         drm_kms_helper_poll_disable(dev);
1652
1653         /* turn off display hw */
1654         list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1655                 drm_helper_connector_dpms(connector, DRM_MODE_DPMS_OFF);
1656         }
1657
1658         /* unpin the front buffers */
1659         list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
1660                 struct amdgpu_framebuffer *rfb = to_amdgpu_framebuffer(crtc->primary->fb);
1661                 struct amdgpu_bo *robj;
1662
1663                 if (rfb == NULL || rfb->obj == NULL) {
1664                         continue;
1665                 }
1666                 robj = gem_to_amdgpu_bo(rfb->obj);
1667                 /* don't unpin kernel fb objects */
1668                 if (!amdgpu_fbdev_robj_is_fb(adev, robj)) {
1669                         r = amdgpu_bo_reserve(robj, false);
1670                         if (r == 0) {
1671                                 amdgpu_bo_unpin(robj);
1672                                 amdgpu_bo_unreserve(robj);
1673                         }
1674                 }
1675         }
1676         /* evict vram memory */
1677         amdgpu_bo_evict_vram(adev);
1678
1679         amdgpu_fence_driver_suspend(adev);
1680
1681         r = amdgpu_suspend(adev);
1682
1683         /* evict remaining vram memory */
1684         amdgpu_bo_evict_vram(adev);
1685
1686         pci_save_state(dev->pdev);
1687         if (suspend) {
1688                 /* Shut down the device */
1689                 pci_disable_device(dev->pdev);
1690                 pci_set_power_state(dev->pdev, PCI_D3hot);
1691         }
1692
1693         if (fbcon) {
1694                 console_lock();
1695                 amdgpu_fbdev_set_suspend(adev, 1);
1696                 console_unlock();
1697         }
1698         return 0;
1699 }
1700
1701 /**
1702  * amdgpu_resume_kms - initiate device resume
1703  *
1704  * @pdev: drm dev pointer
1705  *
1706  * Bring the hw back to operating state (all asics).
1707  * Returns 0 for success or an error on failure.
1708  * Called at driver resume.
1709  */
1710 int amdgpu_resume_kms(struct drm_device *dev, bool resume, bool fbcon)
1711 {
1712         struct drm_connector *connector;
1713         struct amdgpu_device *adev = dev->dev_private;
1714         int r;
1715
1716         if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
1717                 return 0;
1718
1719         if (fbcon) {
1720                 console_lock();
1721         }
1722         if (resume) {
1723                 pci_set_power_state(dev->pdev, PCI_D0);
1724                 pci_restore_state(dev->pdev);
1725                 if (pci_enable_device(dev->pdev)) {
1726                         if (fbcon)
1727                                 console_unlock();
1728                         return -1;
1729                 }
1730         }
1731
1732         /* post card */
1733         amdgpu_atom_asic_init(adev->mode_info.atom_context);
1734
1735         r = amdgpu_resume(adev);
1736
1737         amdgpu_fence_driver_resume(adev);
1738
1739         r = amdgpu_ib_ring_tests(adev);
1740         if (r)
1741                 DRM_ERROR("ib ring test failed (%d).\n", r);
1742
1743         r = amdgpu_late_init(adev);
1744         if (r)
1745                 return r;
1746
1747         /* blat the mode back in */
1748         if (fbcon) {
1749                 drm_helper_resume_force_mode(dev);
1750                 /* turn on display hw */
1751                 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1752                         drm_helper_connector_dpms(connector, DRM_MODE_DPMS_ON);
1753                 }
1754         }
1755
1756         drm_kms_helper_poll_enable(dev);
1757
1758         if (fbcon) {
1759                 amdgpu_fbdev_set_suspend(adev, 0);
1760                 console_unlock();
1761         }
1762
1763         return 0;
1764 }
1765
1766 /**
1767  * amdgpu_gpu_reset - reset the asic
1768  *
1769  * @adev: amdgpu device pointer
1770  *
1771  * Attempt the reset the GPU if it has hung (all asics).
1772  * Returns 0 for success or an error on failure.
1773  */
1774 int amdgpu_gpu_reset(struct amdgpu_device *adev)
1775 {
1776         unsigned ring_sizes[AMDGPU_MAX_RINGS];
1777         uint32_t *ring_data[AMDGPU_MAX_RINGS];
1778
1779         bool saved = false;
1780
1781         int i, r;
1782         int resched;
1783
1784         down_write(&adev->exclusive_lock);
1785
1786         if (!adev->needs_reset) {
1787                 up_write(&adev->exclusive_lock);
1788                 return 0;
1789         }
1790
1791         adev->needs_reset = false;
1792         atomic_inc(&adev->gpu_reset_counter);
1793
1794         /* block TTM */
1795         resched = ttm_bo_lock_delayed_workqueue(&adev->mman.bdev);
1796
1797         r = amdgpu_suspend(adev);
1798
1799         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
1800                 struct amdgpu_ring *ring = adev->rings[i];
1801                 if (!ring)
1802                         continue;
1803
1804                 ring_sizes[i] = amdgpu_ring_backup(ring, &ring_data[i]);
1805                 if (ring_sizes[i]) {
1806                         saved = true;
1807                         dev_info(adev->dev, "Saved %d dwords of commands "
1808                                  "on ring %d.\n", ring_sizes[i], i);
1809                 }
1810         }
1811
1812 retry:
1813         r = amdgpu_asic_reset(adev);
1814         if (!r) {
1815                 dev_info(adev->dev, "GPU reset succeeded, trying to resume\n");
1816                 r = amdgpu_resume(adev);
1817         }
1818
1819         if (!r) {
1820                 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
1821                         struct amdgpu_ring *ring = adev->rings[i];
1822                         if (!ring)
1823                                 continue;
1824
1825                         amdgpu_ring_restore(ring, ring_sizes[i], ring_data[i]);
1826                         ring_sizes[i] = 0;
1827                         ring_data[i] = NULL;
1828                 }
1829
1830                 r = amdgpu_ib_ring_tests(adev);
1831                 if (r) {
1832                         dev_err(adev->dev, "ib ring test failed (%d).\n", r);
1833                         if (saved) {
1834                                 saved = false;
1835                                 r = amdgpu_suspend(adev);
1836                                 goto retry;
1837                         }
1838                 }
1839         } else {
1840                 amdgpu_fence_driver_force_completion(adev);
1841                 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
1842                         if (adev->rings[i])
1843                                 kfree(ring_data[i]);
1844                 }
1845         }
1846
1847         drm_helper_resume_force_mode(adev->ddev);
1848
1849         ttm_bo_unlock_delayed_workqueue(&adev->mman.bdev, resched);
1850         if (r) {
1851                 /* bad news, how to tell it to userspace ? */
1852                 dev_info(adev->dev, "GPU reset failed\n");
1853         }
1854
1855         up_write(&adev->exclusive_lock);
1856         return r;
1857 }
1858
1859
1860 /*
1861  * Debugfs
1862  */
1863 int amdgpu_debugfs_add_files(struct amdgpu_device *adev,
1864                              struct drm_info_list *files,
1865                              unsigned nfiles)
1866 {
1867         unsigned i;
1868
1869         for (i = 0; i < adev->debugfs_count; i++) {
1870                 if (adev->debugfs[i].files == files) {
1871                         /* Already registered */
1872                         return 0;
1873                 }
1874         }
1875
1876         i = adev->debugfs_count + 1;
1877         if (i > AMDGPU_DEBUGFS_MAX_COMPONENTS) {
1878                 DRM_ERROR("Reached maximum number of debugfs components.\n");
1879                 DRM_ERROR("Report so we increase "
1880                           "AMDGPU_DEBUGFS_MAX_COMPONENTS.\n");
1881                 return -EINVAL;
1882         }
1883         adev->debugfs[adev->debugfs_count].files = files;
1884         adev->debugfs[adev->debugfs_count].num_files = nfiles;
1885         adev->debugfs_count = i;
1886 #if defined(CONFIG_DEBUG_FS)
1887         drm_debugfs_create_files(files, nfiles,
1888                                  adev->ddev->control->debugfs_root,
1889                                  adev->ddev->control);
1890         drm_debugfs_create_files(files, nfiles,
1891                                  adev->ddev->primary->debugfs_root,
1892                                  adev->ddev->primary);
1893 #endif
1894         return 0;
1895 }
1896
1897 static void amdgpu_debugfs_remove_files(struct amdgpu_device *adev)
1898 {
1899 #if defined(CONFIG_DEBUG_FS)
1900         unsigned i;
1901
1902         for (i = 0; i < adev->debugfs_count; i++) {
1903                 drm_debugfs_remove_files(adev->debugfs[i].files,
1904                                          adev->debugfs[i].num_files,
1905                                          adev->ddev->control);
1906                 drm_debugfs_remove_files(adev->debugfs[i].files,
1907                                          adev->debugfs[i].num_files,
1908                                          adev->ddev->primary);
1909         }
1910 #endif
1911 }
1912
1913 #if defined(CONFIG_DEBUG_FS)
1914
1915 static ssize_t amdgpu_debugfs_regs_read(struct file *f, char __user *buf,
1916                                         size_t size, loff_t *pos)
1917 {
1918         struct amdgpu_device *adev = f->f_inode->i_private;
1919         ssize_t result = 0;
1920         int r;
1921
1922         if (size & 0x3 || *pos & 0x3)
1923                 return -EINVAL;
1924
1925         while (size) {
1926                 uint32_t value;
1927
1928                 if (*pos > adev->rmmio_size)
1929                         return result;
1930
1931                 value = RREG32(*pos >> 2);
1932                 r = put_user(value, (uint32_t *)buf);
1933                 if (r)
1934                         return r;
1935
1936                 result += 4;
1937                 buf += 4;
1938                 *pos += 4;
1939                 size -= 4;
1940         }
1941
1942         return result;
1943 }
1944
1945 static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,
1946                                          size_t size, loff_t *pos)
1947 {
1948         struct amdgpu_device *adev = f->f_inode->i_private;
1949         ssize_t result = 0;
1950         int r;
1951
1952         if (size & 0x3 || *pos & 0x3)
1953                 return -EINVAL;
1954
1955         while (size) {
1956                 uint32_t value;
1957
1958                 if (*pos > adev->rmmio_size)
1959                         return result;
1960
1961                 r = get_user(value, (uint32_t *)buf);
1962                 if (r)
1963                         return r;
1964
1965                 WREG32(*pos >> 2, value);
1966
1967                 result += 4;
1968                 buf += 4;
1969                 *pos += 4;
1970                 size -= 4;
1971         }
1972
1973         return result;
1974 }
1975
1976 static const struct file_operations amdgpu_debugfs_regs_fops = {
1977         .owner = THIS_MODULE,
1978         .read = amdgpu_debugfs_regs_read,
1979         .write = amdgpu_debugfs_regs_write,
1980         .llseek = default_llseek
1981 };
1982
1983 static int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
1984 {
1985         struct drm_minor *minor = adev->ddev->primary;
1986         struct dentry *ent, *root = minor->debugfs_root;
1987
1988         ent = debugfs_create_file("amdgpu_regs", S_IFREG | S_IRUGO, root,
1989                                   adev, &amdgpu_debugfs_regs_fops);
1990         if (IS_ERR(ent))
1991                 return PTR_ERR(ent);
1992         i_size_write(ent->d_inode, adev->rmmio_size);
1993         adev->debugfs_regs = ent;
1994
1995         return 0;
1996 }
1997
1998 static void amdgpu_debugfs_regs_cleanup(struct amdgpu_device *adev)
1999 {
2000         debugfs_remove(adev->debugfs_regs);
2001         adev->debugfs_regs = NULL;
2002 }
2003
2004 int amdgpu_debugfs_init(struct drm_minor *minor)
2005 {
2006         return 0;
2007 }
2008
2009 void amdgpu_debugfs_cleanup(struct drm_minor *minor)
2010 {
2011 }
2012 #else
2013 static int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
2014 {
2015         return 0;
2016 }
2017 static void amdgpu_debugfs_regs_cleanup(struct amdgpu_device *adev) { }
2018 #endif