drivers/gpu/drm/amd/amdgpu/amdgpu_device.c

   1 /*
   2  * Copyright 2008 Advanced Micro Devices, Inc.
   3  * Copyright 2008 Red Hat Inc.
   4  * Copyright 2009 Jerome Glisse.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22  * OTHER DEALINGS IN THE SOFTWARE.
  23  *
  24  * Authors: Dave Airlie
  25  *          Alex Deucher
  26  *          Jerome Glisse
  27  */
  28 #include <linux/kthread.h>
  29 #include <linux/console.h>
  30 #include <linux/slab.h>
  31 #include <linux/debugfs.h>
  32 #include <drm/drmP.h>
  33 #include <drm/drm_crtc_helper.h>
  34 #include <drm/amdgpu_drm.h>
  35 #include <linux/vgaarb.h>
  36 #include <linux/vga_switcheroo.h>
  37 #include <linux/efi.h>
  38 #include "amdgpu.h"
  39 #include "amdgpu_trace.h"
  40 #include "amdgpu_i2c.h"
  41 #include "atom.h"
  42 #include "amdgpu_atombios.h"
  43 #include "amd_pcie.h"
  44 #ifdef CONFIG_DRM_AMDGPU_SI
  45 #include "si.h"
  46 #endif
  47 #ifdef CONFIG_DRM_AMDGPU_CIK
  48 #include "cik.h"
  49 #endif
  50 #include "vi.h"
  51 #include "bif/bif_4_1_d.h"
  52 #include <linux/pci.h>
  53 #include <linux/firmware.h>
  54
  55 static int amdgpu_debugfs_regs_init(struct amdgpu_device *adev);
  56 static void amdgpu_debugfs_regs_cleanup(struct amdgpu_device *adev);
  57
  58 static const char *amdgpu_asic_name[] = {
  59         "TAHITI",
  60         "PITCAIRN",
  61         "VERDE",
  62         "OLAND",
  63         "HAINAN",
  64         "BONAIRE",
  65         "KAVERI",
  66         "KABINI",
  67         "HAWAII",
  68         "MULLINS",
  69         "TOPAZ",
  70         "TONGA",
  71         "FIJI",
  72         "CARRIZO",
  73         "STONEY",
  74         "POLARIS10",
  75         "POLARIS11",
  76         "POLARIS12",
  77         "LAST",
  78 };
  79
  80 bool amdgpu_device_is_px(struct drm_device *dev)
  81 {
  82         struct amdgpu_device *adev = dev->dev_private;
  83
  84         if (adev->flags & AMD_IS_PX)
  85                 return true;
  86         return false;
  87 }
  88
  89 /*
  90  * MMIO register access helper functions.
  91  */
  92 uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg,
  93                         bool always_indirect)
  94 {
  95         uint32_t ret;
  96
  97         if (amdgpu_sriov_runtime(adev)) {
  98                 BUG_ON(in_interrupt());
  99                 return amdgpu_virt_kiq_rreg(adev, reg);
 100         }
 101
 102         if ((reg * 4) < adev->rmmio_size && !always_indirect)
 103                 ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
 104         else {
 105                 unsigned long flags;
 106
 107                 spin_lock_irqsave(&adev->mmio_idx_lock, flags);
 108                 writel((reg * 4), ((void __iomem *)adev->rmmio) + (mmMM_INDEX * 4));
 109                 ret = readl(((void __iomem *)adev->rmmio) + (mmMM_DATA * 4));
 110                 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
 111         }
 112         trace_amdgpu_mm_rreg(adev->pdev->device, reg, ret);
 113         return ret;
 114 }
 115
 116 void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v,
 117                     bool always_indirect)
 118 {
 119         trace_amdgpu_mm_wreg(adev->pdev->device, reg, v);
 120
 121         if (amdgpu_sriov_runtime(adev)) {
 122                 BUG_ON(in_interrupt());
 123                 return amdgpu_virt_kiq_wreg(adev, reg, v);
 124         }
 125
 126         if ((reg * 4) < adev->rmmio_size && !always_indirect)
 127                 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
 128         else {
 129                 unsigned long flags;
 130
 131                 spin_lock_irqsave(&adev->mmio_idx_lock, flags);
 132                 writel((reg * 4), ((void __iomem *)adev->rmmio) + (mmMM_INDEX * 4));
 133                 writel(v, ((void __iomem *)adev->rmmio) + (mmMM_DATA * 4));
 134                 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
 135         }
 136 }
 137
 138 u32 amdgpu_io_rreg(struct amdgpu_device *adev, u32 reg)
 139 {
 140         if ((reg * 4) < adev->rio_mem_size)
 141                 return ioread32(adev->rio_mem + (reg * 4));
 142         else {
 143                 iowrite32((reg * 4), adev->rio_mem + (mmMM_INDEX * 4));
 144                 return ioread32(adev->rio_mem + (mmMM_DATA * 4));
 145         }
 146 }
 147
 148 void amdgpu_io_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 149 {
 150
 151         if ((reg * 4) < adev->rio_mem_size)
 152                 iowrite32(v, adev->rio_mem + (reg * 4));
 153         else {
 154                 iowrite32((reg * 4), adev->rio_mem + (mmMM_INDEX * 4));
 155                 iowrite32(v, adev->rio_mem + (mmMM_DATA * 4));
 156         }
 157 }
 158
 159 /**
 160  * amdgpu_mm_rdoorbell - read a doorbell dword
 161  *
 162  * @adev: amdgpu_device pointer
 163  * @index: doorbell index
 164  *
 165  * Returns the value in the doorbell aperture at the
 166  * requested doorbell index (CIK).
 167  */
 168 u32 amdgpu_mm_rdoorbell(struct amdgpu_device *adev, u32 index)
 169 {
 170         if (index < adev->doorbell.num_doorbells) {
 171                 return readl(adev->doorbell.ptr + index);
 172         } else {
 173                 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
 174                 return 0;
 175         }
 176 }
 177
 178 /**
 179  * amdgpu_mm_wdoorbell - write a doorbell dword
 180  *
 181  * @adev: amdgpu_device pointer
 182  * @index: doorbell index
 183  * @v: value to write
 184  *
 185  * Writes @v to the doorbell aperture at the
 186  * requested doorbell index (CIK).
 187  */
 188 void amdgpu_mm_wdoorbell(struct amdgpu_device *adev, u32 index, u32 v)
 189 {
 190         if (index < adev->doorbell.num_doorbells) {
 191                 writel(v, adev->doorbell.ptr + index);
 192         } else {
 193                 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
 194         }
 195 }
 196
 197 /**
 198  * amdgpu_invalid_rreg - dummy reg read function
 199  *
 200  * @adev: amdgpu device pointer
 201  * @reg: offset of register
 202  *
 203  * Dummy register read function.  Used for register blocks
 204  * that certain asics don't have (all asics).
 205  * Returns the value in the register.
 206  */
 207 static uint32_t amdgpu_invalid_rreg(struct amdgpu_device *adev, uint32_t reg)
 208 {
 209         DRM_ERROR("Invalid callback to read register 0x%04X\n", reg);
 210         BUG();
 211         return 0;
 212 }
 213
 214 /**
 215  * amdgpu_invalid_wreg - dummy reg write function
 216  *
 217  * @adev: amdgpu device pointer
 218  * @reg: offset of register
 219  * @v: value to write to the register
 220  *
 221  * Dummy register read function.  Used for register blocks
 222  * that certain asics don't have (all asics).
 223  */
 224 static void amdgpu_invalid_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
 225 {
 226         DRM_ERROR("Invalid callback to write register 0x%04X with 0x%08X\n",
 227                   reg, v);
 228         BUG();
 229 }
 230
 231 /**
 232  * amdgpu_block_invalid_rreg - dummy reg read function
 233  *
 234  * @adev: amdgpu device pointer
 235  * @block: offset of instance
 236  * @reg: offset of register
 237  *
 238  * Dummy register read function.  Used for register blocks
 239  * that certain asics don't have (all asics).
 240  * Returns the value in the register.
 241  */
 242 static uint32_t amdgpu_block_invalid_rreg(struct amdgpu_device *adev,
 243                                           uint32_t block, uint32_t reg)
 244 {
 245         DRM_ERROR("Invalid callback to read register 0x%04X in block 0x%04X\n",
 246                   reg, block);
 247         BUG();
 248         return 0;
 249 }
 250
 251 /**
 252  * amdgpu_block_invalid_wreg - dummy reg write function
 253  *
 254  * @adev: amdgpu device pointer
 255  * @block: offset of instance
 256  * @reg: offset of register
 257  * @v: value to write to the register
 258  *
 259  * Dummy register read function.  Used for register blocks
 260  * that certain asics don't have (all asics).
 261  */
 262 static void amdgpu_block_invalid_wreg(struct amdgpu_device *adev,
 263                                       uint32_t block,
 264                                       uint32_t reg, uint32_t v)
 265 {
 266         DRM_ERROR("Invalid block callback to write register 0x%04X in block 0x%04X with 0x%08X\n",
 267                   reg, block, v);
 268         BUG();
 269 }
 270
 271 static int amdgpu_vram_scratch_init(struct amdgpu_device *adev)
 272 {
 273         int r;
 274
 275         if (adev->vram_scratch.robj == NULL) {
 276                 r = amdgpu_bo_create(adev, AMDGPU_GPU_PAGE_SIZE,
 277                                      PAGE_SIZE, true, AMDGPU_GEM_DOMAIN_VRAM,
 278                                      AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
 279                                      AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS,
 280                                      NULL, NULL, &adev->vram_scratch.robj);
 281                 if (r) {
 282                         return r;
 283                 }
 284         }
 285
 286         r = amdgpu_bo_reserve(adev->vram_scratch.robj, false);
 287         if (unlikely(r != 0))
 288                 return r;
 289         r = amdgpu_bo_pin(adev->vram_scratch.robj,
 290                           AMDGPU_GEM_DOMAIN_VRAM, &adev->vram_scratch.gpu_addr);
 291         if (r) {
 292                 amdgpu_bo_unreserve(adev->vram_scratch.robj);
 293                 return r;
 294         }
 295         r = amdgpu_bo_kmap(adev->vram_scratch.robj,
 296                                 (void **)&adev->vram_scratch.ptr);
 297         if (r)
 298                 amdgpu_bo_unpin(adev->vram_scratch.robj);
 299         amdgpu_bo_unreserve(adev->vram_scratch.robj);
 300
 301         return r;
 302 }
 303
 304 static void amdgpu_vram_scratch_fini(struct amdgpu_device *adev)
 305 {
 306         int r;
 307
 308         if (adev->vram_scratch.robj == NULL) {
 309                 return;
 310         }
 311         r = amdgpu_bo_reserve(adev->vram_scratch.robj, false);
 312         if (likely(r == 0)) {
 313                 amdgpu_bo_kunmap(adev->vram_scratch.robj);
 314                 amdgpu_bo_unpin(adev->vram_scratch.robj);
 315                 amdgpu_bo_unreserve(adev->vram_scratch.robj);
 316         }
 317         amdgpu_bo_unref(&adev->vram_scratch.robj);
 318 }
 319
 320 /**
 321  * amdgpu_program_register_sequence - program an array of registers.
 322  *
 323  * @adev: amdgpu_device pointer
 324  * @registers: pointer to the register array
 325  * @array_size: size of the register array
 326  *
 327  * Programs an array or registers with and and or masks.
 328  * This is a helper for setting golden registers.
 329  */
 330 void amdgpu_program_register_sequence(struct amdgpu_device *adev,
 331                                       const u32 *registers,
 332                                       const u32 array_size)
 333 {
 334         u32 tmp, reg, and_mask, or_mask;
 335         int i;
 336
 337         if (array_size % 3)
 338                 return;
 339
 340         for (i = 0; i < array_size; i +=3) {
 341                 reg = registers[i + 0];
 342                 and_mask = registers[i + 1];
 343                 or_mask = registers[i + 2];
 344
 345                 if (and_mask == 0xffffffff) {
 346                         tmp = or_mask;
 347                 } else {
 348                         tmp = RREG32(reg);
 349                         tmp &= ~and_mask;
 350                         tmp |= or_mask;
 351                 }
 352                 WREG32(reg, tmp);
 353         }
 354 }
 355
 356 void amdgpu_pci_config_reset(struct amdgpu_device *adev)
 357 {
 358         pci_write_config_dword(adev->pdev, 0x7c, AMDGPU_ASIC_RESET_DATA);
 359 }
 360
 361 /*
 362  * GPU doorbell aperture helpers function.
 363  */
 364 /**
 365  * amdgpu_doorbell_init - Init doorbell driver information.
 366  *
 367  * @adev: amdgpu_device pointer
 368  *
 369  * Init doorbell driver information (CIK)
 370  * Returns 0 on success, error on failure.
 371  */
 372 static int amdgpu_doorbell_init(struct amdgpu_device *adev)
 373 {
 374         /* doorbell bar mapping */
 375         adev->doorbell.base = pci_resource_start(adev->pdev, 2);
 376         adev->doorbell.size = pci_resource_len(adev->pdev, 2);
 377
 378         adev->doorbell.num_doorbells = min_t(u32, adev->doorbell.size / sizeof(u32),
 379                                              AMDGPU_DOORBELL_MAX_ASSIGNMENT+1);
 380         if (adev->doorbell.num_doorbells == 0)
 381                 return -EINVAL;
 382
 383         adev->doorbell.ptr = ioremap(adev->doorbell.base, adev->doorbell.num_doorbells * sizeof(u32));
 384         if (adev->doorbell.ptr == NULL) {
 385                 return -ENOMEM;
 386         }
 387         DRM_INFO("doorbell mmio base: 0x%08X\n", (uint32_t)adev->doorbell.base);
 388         DRM_INFO("doorbell mmio size: %u\n", (unsigned)adev->doorbell.size);
 389
 390         return 0;
 391 }
 392
 393 /**
 394  * amdgpu_doorbell_fini - Tear down doorbell driver information.
 395  *
 396  * @adev: amdgpu_device pointer
 397  *
 398  * Tear down doorbell driver information (CIK)
 399  */
 400 static void amdgpu_doorbell_fini(struct amdgpu_device *adev)
 401 {
 402         iounmap(adev->doorbell.ptr);
 403         adev->doorbell.ptr = NULL;
 404 }
 405
 406 /**
 407  * amdgpu_doorbell_get_kfd_info - Report doorbell configuration required to
 408  *                                setup amdkfd
 409  *
 410  * @adev: amdgpu_device pointer
 411  * @aperture_base: output returning doorbell aperture base physical address
 412  * @aperture_size: output returning doorbell aperture size in bytes
 413  * @start_offset: output returning # of doorbell bytes reserved for amdgpu.
 414  *
 415  * amdgpu and amdkfd share the doorbell aperture. amdgpu sets it up,
 416  * takes doorbells required for its own rings and reports the setup to amdkfd.
 417  * amdgpu reserved doorbells are at the start of the doorbell aperture.
 418  */
 419 void amdgpu_doorbell_get_kfd_info(struct amdgpu_device *adev,
 420                                 phys_addr_t *aperture_base,
 421                                 size_t *aperture_size,
 422                                 size_t *start_offset)
 423 {
 424         /*
 425          * The first num_doorbells are used by amdgpu.
 426          * amdkfd takes whatever's left in the aperture.
 427          */
 428         if (adev->doorbell.size > adev->doorbell.num_doorbells * sizeof(u32)) {
 429                 *aperture_base = adev->doorbell.base;
 430                 *aperture_size = adev->doorbell.size;
 431                 *start_offset = adev->doorbell.num_doorbells * sizeof(u32);
 432         } else {
 433                 *aperture_base = 0;
 434                 *aperture_size = 0;
 435                 *start_offset = 0;
 436         }
 437 }
 438
 439 /*
 440  * amdgpu_wb_*()
 441  * Writeback is the the method by which the the GPU updates special pages
 442  * in memory with the status of certain GPU events (fences, ring pointers,
 443  * etc.).
 444  */
 445
 446 /**
 447  * amdgpu_wb_fini - Disable Writeback and free memory
 448  *
 449  * @adev: amdgpu_device pointer
 450  *
 451  * Disables Writeback and frees the Writeback memory (all asics).
 452  * Used at driver shutdown.
 453  */
 454 static void amdgpu_wb_fini(struct amdgpu_device *adev)
 455 {
 456         if (adev->wb.wb_obj) {
 457                 amdgpu_bo_free_kernel(&adev->wb.wb_obj,
 458                                       &adev->wb.gpu_addr,
 459                                       (void **)&adev->wb.wb);
 460                 adev->wb.wb_obj = NULL;
 461         }
 462 }
 463
 464 /**
 465  * amdgpu_wb_init- Init Writeback driver info and allocate memory
 466  *
 467  * @adev: amdgpu_device pointer
 468  *
 469  * Disables Writeback and frees the Writeback memory (all asics).
 470  * Used at driver startup.
 471  * Returns 0 on success or an -error on failure.
 472  */
 473 static int amdgpu_wb_init(struct amdgpu_device *adev)
 474 {
 475         int r;
 476
 477         if (adev->wb.wb_obj == NULL) {
 478                 r = amdgpu_bo_create_kernel(adev, AMDGPU_MAX_WB * 4,
 479                                             PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
 480                                             &adev->wb.wb_obj, &adev->wb.gpu_addr,
 481                                             (void **)&adev->wb.wb);
 482                 if (r) {
 483                         dev_warn(adev->dev, "(%d) create WB bo failed\n", r);
 484                         return r;
 485                 }
 486
 487                 adev->wb.num_wb = AMDGPU_MAX_WB;
 488                 memset(&adev->wb.used, 0, sizeof(adev->wb.used));
 489
 490                 /* clear wb memory */
 491                 memset((char *)adev->wb.wb, 0, AMDGPU_GPU_PAGE_SIZE);
 492         }
 493
 494         return 0;
 495 }
 496
 497 /**
 498  * amdgpu_wb_get - Allocate a wb entry
 499  *
 500  * @adev: amdgpu_device pointer
 501  * @wb: wb index
 502  *
 503  * Allocate a wb slot for use by the driver (all asics).
 504  * Returns 0 on success or -EINVAL on failure.
 505  */
 506 int amdgpu_wb_get(struct amdgpu_device *adev, u32 *wb)
 507 {
 508         unsigned long offset = find_first_zero_bit(adev->wb.used, adev->wb.num_wb);
 509         if (offset < adev->wb.num_wb) {
 510                 __set_bit(offset, adev->wb.used);
 511                 *wb = offset;
 512                 return 0;
 513         } else {
 514                 return -EINVAL;
 515         }
 516 }
 517
 518 /**
 519  * amdgpu_wb_free - Free a wb entry
 520  *
 521  * @adev: amdgpu_device pointer
 522  * @wb: wb index
 523  *
 524  * Free a wb slot allocated for use by the driver (all asics)
 525  */
 526 void amdgpu_wb_free(struct amdgpu_device *adev, u32 wb)
 527 {
 528         if (wb < adev->wb.num_wb)
 529                 __clear_bit(wb, adev->wb.used);
 530 }
 531
 532 /**
 533  * amdgpu_vram_location - try to find VRAM location
 534  * @adev: amdgpu device structure holding all necessary informations
 535  * @mc: memory controller structure holding memory informations
 536  * @base: base address at which to put VRAM
 537  *
 538  * Function will place try to place VRAM at base address provided
 539  * as parameter (which is so far either PCI aperture address or
 540  * for IGP TOM base address).
 541  *
 542  * If there is not enough space to fit the unvisible VRAM in the 32bits
 543  * address space then we limit the VRAM size to the aperture.
 544  *
 545  * Note: We don't explicitly enforce VRAM start to be aligned on VRAM size,
 546  * this shouldn't be a problem as we are using the PCI aperture as a reference.
 547  * Otherwise this would be needed for rv280, all r3xx, and all r4xx, but
 548  * not IGP.
 549  *
 550  * Note: we use mc_vram_size as on some board we need to program the mc to
 551  * cover the whole aperture even if VRAM size is inferior to aperture size
 552  * Novell bug 204882 + along with lots of ubuntu ones
 553  *
 554  * Note: when limiting vram it's safe to overwritte real_vram_size because
 555  * we are not in case where real_vram_size is inferior to mc_vram_size (ie
 556  * note afected by bogus hw of Novell bug 204882 + along with lots of ubuntu
 557  * ones)
 558  *
 559  * Note: IGP TOM addr should be the same as the aperture addr, we don't
 560  * explicitly check for that thought.
 561  *
 562  * FIXME: when reducing VRAM size align new size on power of 2.
 563  */
 564 void amdgpu_vram_location(struct amdgpu_device *adev, struct amdgpu_mc *mc, u64 base)
 565 {
 566         uint64_t limit = (uint64_t)amdgpu_vram_limit << 20;
 567
 568         mc->vram_start = base;
 569         if (mc->mc_vram_size > (adev->mc.mc_mask - base + 1)) {
 570                 dev_warn(adev->dev, "limiting VRAM to PCI aperture size\n");
 571                 mc->real_vram_size = mc->aper_size;
 572                 mc->mc_vram_size = mc->aper_size;
 573         }
 574         mc->vram_end = mc->vram_start + mc->mc_vram_size - 1;
 575         if (limit && limit < mc->real_vram_size)
 576                 mc->real_vram_size = limit;
 577         dev_info(adev->dev, "VRAM: %lluM 0x%016llX - 0x%016llX (%lluM used)\n",
 578                         mc->mc_vram_size >> 20, mc->vram_start,
 579                         mc->vram_end, mc->real_vram_size >> 20);
 580 }
 581
 582 /**
 583  * amdgpu_gtt_location - try to find GTT location
 584  * @adev: amdgpu device structure holding all necessary informations
 585  * @mc: memory controller structure holding memory informations
 586  *
 587  * Function will place try to place GTT before or after VRAM.
 588  *
 589  * If GTT size is bigger than space left then we ajust GTT size.
 590  * Thus function will never fails.
 591  *
 592  * FIXME: when reducing GTT size align new size on power of 2.
 593  */
 594 void amdgpu_gtt_location(struct amdgpu_device *adev, struct amdgpu_mc *mc)
 595 {
 596         u64 size_af, size_bf;
 597
 598         size_af = ((adev->mc.mc_mask - mc->vram_end) + mc->gtt_base_align) & ~mc->gtt_base_align;
 599         size_bf = mc->vram_start & ~mc->gtt_base_align;
 600         if (size_bf > size_af) {
 601                 if (mc->gtt_size > size_bf) {
 602                         dev_warn(adev->dev, "limiting GTT\n");
 603                         mc->gtt_size = size_bf;
 604                 }
 605                 mc->gtt_start = (mc->vram_start & ~mc->gtt_base_align) - mc->gtt_size;
 606         } else {
 607                 if (mc->gtt_size > size_af) {
 608                         dev_warn(adev->dev, "limiting GTT\n");
 609                         mc->gtt_size = size_af;
 610                 }
 611                 mc->gtt_start = (mc->vram_end + 1 + mc->gtt_base_align) & ~mc->gtt_base_align;
 612         }
 613         mc->gtt_end = mc->gtt_start + mc->gtt_size - 1;
 614         dev_info(adev->dev, "GTT: %lluM 0x%016llX - 0x%016llX\n",
 615                         mc->gtt_size >> 20, mc->gtt_start, mc->gtt_end);
 616 }
 617
 618 /*
 619  * GPU helpers function.
 620  */
 621 /**
 622  * amdgpu_card_posted - check if the hw has already been initialized
 623  *
 624  * @adev: amdgpu_device pointer
 625  *
 626  * Check if the asic has been initialized (all asics).
 627  * Used at driver startup.
 628  * Returns true if initialized or false if not.
 629  */
 630 bool amdgpu_card_posted(struct amdgpu_device *adev)
 631 {
 632         uint32_t reg;
 633
 634         /* then check MEM_SIZE, in case the crtcs are off */
 635         reg = RREG32(mmCONFIG_MEMSIZE);
 636
 637         if (reg)
 638                 return true;
 639
 640         return false;
 641
 642 }
 643
 644 static bool amdgpu_vpost_needed(struct amdgpu_device *adev)
 645 {
 646         if (amdgpu_sriov_vf(adev))
 647                 return false;
 648
 649         if (amdgpu_passthrough(adev)) {
 650                 /* for FIJI: In whole GPU pass-through virtualization case, after VM reboot
 651                  * some old smc fw still need driver do vPost otherwise gpu hang, while
 652                  * those smc fw version above 22.15 doesn't have this flaw, so we force
 653                  * vpost executed for smc version below 22.15
 654                  */
 655                 if (adev->asic_type == CHIP_FIJI) {
 656                         int err;
 657                         uint32_t fw_ver;
 658                         err = request_firmware(&adev->pm.fw, "amdgpu/fiji_smc.bin", adev->dev);
 659                         /* force vPost if error occured */
 660                         if (err)
 661                                 return true;
 662
 663                         fw_ver = *((uint32_t *)adev->pm.fw->data + 69);
 664                         if (fw_ver < 0x00160e00)
 665                                 return true;
 666                 }
 667         }
 668         return !amdgpu_card_posted(adev);
 669 }
 670
 671 /**
 672  * amdgpu_dummy_page_init - init dummy page used by the driver
 673  *
 674  * @adev: amdgpu_device pointer
 675  *
 676  * Allocate the dummy page used by the driver (all asics).
 677  * This dummy page is used by the driver as a filler for gart entries
 678  * when pages are taken out of the GART
 679  * Returns 0 on sucess, -ENOMEM on failure.
 680  */
 681 int amdgpu_dummy_page_init(struct amdgpu_device *adev)
 682 {
 683         if (adev->dummy_page.page)
 684                 return 0;
 685         adev->dummy_page.page = alloc_page(GFP_DMA32 | GFP_KERNEL | __GFP_ZERO);
 686         if (adev->dummy_page.page == NULL)
 687                 return -ENOMEM;
 688         adev->dummy_page.addr = pci_map_page(adev->pdev, adev->dummy_page.page,
 689                                         0, PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
 690         if (pci_dma_mapping_error(adev->pdev, adev->dummy_page.addr)) {
 691                 dev_err(&adev->pdev->dev, "Failed to DMA MAP the dummy page\n");
 692                 __free_page(adev->dummy_page.page);
 693                 adev->dummy_page.page = NULL;
 694                 return -ENOMEM;
 695         }
 696         return 0;
 697 }
 698
 699 /**
 700  * amdgpu_dummy_page_fini - free dummy page used by the driver
 701  *
 702  * @adev: amdgpu_device pointer
 703  *
 704  * Frees the dummy page used by the driver (all asics).
 705  */
 706 void amdgpu_dummy_page_fini(struct amdgpu_device *adev)
 707 {
 708         if (adev->dummy_page.page == NULL)
 709                 return;
 710         pci_unmap_page(adev->pdev, adev->dummy_page.addr,
 711                         PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
 712         __free_page(adev->dummy_page.page);
 713         adev->dummy_page.page = NULL;
 714 }
 715
 716
 717 /* ATOM accessor methods */
 718 /*
 719  * ATOM is an interpreted byte code stored in tables in the vbios.  The
 720  * driver registers callbacks to access registers and the interpreter
 721  * in the driver parses the tables and executes then to program specific
 722  * actions (set display modes, asic init, etc.).  See amdgpu_atombios.c,
 723  * atombios.h, and atom.c
 724  */
 725
 726 /**
 727  * cail_pll_read - read PLL register
 728  *
 729  * @info: atom card_info pointer
 730  * @reg: PLL register offset
 731  *
 732  * Provides a PLL register accessor for the atom interpreter (r4xx+).
 733  * Returns the value of the PLL register.
 734  */
 735 static uint32_t cail_pll_read(struct card_info *info, uint32_t reg)
 736 {
 737         return 0;
 738 }
 739
 740 /**
 741  * cail_pll_write - write PLL register
 742  *
 743  * @info: atom card_info pointer
 744  * @reg: PLL register offset
 745  * @val: value to write to the pll register
 746  *
 747  * Provides a PLL register accessor for the atom interpreter (r4xx+).
 748  */
 749 static void cail_pll_write(struct card_info *info, uint32_t reg, uint32_t val)
 750 {
 751
 752 }
 753
 754 /**
 755  * cail_mc_read - read MC (Memory Controller) register
 756  *
 757  * @info: atom card_info pointer
 758  * @reg: MC register offset
 759  *
 760  * Provides an MC register accessor for the atom interpreter (r4xx+).
 761  * Returns the value of the MC register.
 762  */
 763 static uint32_t cail_mc_read(struct card_info *info, uint32_t reg)
 764 {
 765         return 0;
 766 }
 767
 768 /**
 769  * cail_mc_write - write MC (Memory Controller) register
 770  *
 771  * @info: atom card_info pointer
 772  * @reg: MC register offset
 773  * @val: value to write to the pll register
 774  *
 775  * Provides a MC register accessor for the atom interpreter (r4xx+).
 776  */
 777 static void cail_mc_write(struct card_info *info, uint32_t reg, uint32_t val)
 778 {
 779
 780 }
 781
 782 /**
 783  * cail_reg_write - write MMIO register
 784  *
 785  * @info: atom card_info pointer
 786  * @reg: MMIO register offset
 787  * @val: value to write to the pll register
 788  *
 789  * Provides a MMIO register accessor for the atom interpreter (r4xx+).
 790  */
 791 static void cail_reg_write(struct card_info *info, uint32_t reg, uint32_t val)
 792 {
 793         struct amdgpu_device *adev = info->dev->dev_private;
 794
 795         WREG32(reg, val);
 796 }
 797
 798 /**
 799  * cail_reg_read - read MMIO register
 800  *
 801  * @info: atom card_info pointer
 802  * @reg: MMIO register offset
 803  *
 804  * Provides an MMIO register accessor for the atom interpreter (r4xx+).
 805  * Returns the value of the MMIO register.
 806  */
 807 static uint32_t cail_reg_read(struct card_info *info, uint32_t reg)
 808 {
 809         struct amdgpu_device *adev = info->dev->dev_private;
 810         uint32_t r;
 811
 812         r = RREG32(reg);
 813         return r;
 814 }
 815
 816 /**
 817  * cail_ioreg_write - write IO register
 818  *
 819  * @info: atom card_info pointer
 820  * @reg: IO register offset
 821  * @val: value to write to the pll register
 822  *
 823  * Provides a IO register accessor for the atom interpreter (r4xx+).
 824  */
 825 static void cail_ioreg_write(struct card_info *info, uint32_t reg, uint32_t val)
 826 {
 827         struct amdgpu_device *adev = info->dev->dev_private;
 828
 829         WREG32_IO(reg, val);
 830 }
 831
 832 /**
 833  * cail_ioreg_read - read IO register
 834  *
 835  * @info: atom card_info pointer
 836  * @reg: IO register offset
 837  *
 838  * Provides an IO register accessor for the atom interpreter (r4xx+).
 839  * Returns the value of the IO register.
 840  */
 841 static uint32_t cail_ioreg_read(struct card_info *info, uint32_t reg)
 842 {
 843         struct amdgpu_device *adev = info->dev->dev_private;
 844         uint32_t r;
 845
 846         r = RREG32_IO(reg);
 847         return r;
 848 }
 849
 850 /**
 851  * amdgpu_atombios_fini - free the driver info and callbacks for atombios
 852  *
 853  * @adev: amdgpu_device pointer
 854  *
 855  * Frees the driver info and register access callbacks for the ATOM
 856  * interpreter (r4xx+).
 857  * Called at driver shutdown.
 858  */
 859 static void amdgpu_atombios_fini(struct amdgpu_device *adev)
 860 {
 861         if (adev->mode_info.atom_context) {
 862                 kfree(adev->mode_info.atom_context->scratch);
 863                 kfree(adev->mode_info.atom_context->iio);
 864         }
 865         kfree(adev->mode_info.atom_context);
 866         adev->mode_info.atom_context = NULL;
 867         kfree(adev->mode_info.atom_card_info);
 868         adev->mode_info.atom_card_info = NULL;
 869 }
 870
 871 /**
 872  * amdgpu_atombios_init - init the driver info and callbacks for atombios
 873  *
 874  * @adev: amdgpu_device pointer
 875  *
 876  * Initializes the driver info and register access callbacks for the
 877  * ATOM interpreter (r4xx+).
 878  * Returns 0 on sucess, -ENOMEM on failure.
 879  * Called at driver startup.
 880  */
 881 static int amdgpu_atombios_init(struct amdgpu_device *adev)
 882 {
 883         struct card_info *atom_card_info =
 884             kzalloc(sizeof(struct card_info), GFP_KERNEL);
 885
 886         if (!atom_card_info)
 887                 return -ENOMEM;
 888
 889         adev->mode_info.atom_card_info = atom_card_info;
 890         atom_card_info->dev = adev->ddev;
 891         atom_card_info->reg_read = cail_reg_read;
 892         atom_card_info->reg_write = cail_reg_write;
 893         /* needed for iio ops */
 894         if (adev->rio_mem) {
 895                 atom_card_info->ioreg_read = cail_ioreg_read;
 896                 atom_card_info->ioreg_write = cail_ioreg_write;
 897         } else {
 898                 DRM_INFO("PCI I/O BAR is not found. Using MMIO to access ATOM BIOS\n");
 899                 atom_card_info->ioreg_read = cail_reg_read;
 900                 atom_card_info->ioreg_write = cail_reg_write;
 901         }
 902         atom_card_info->mc_read = cail_mc_read;
 903         atom_card_info->mc_write = cail_mc_write;
 904         atom_card_info->pll_read = cail_pll_read;
 905         atom_card_info->pll_write = cail_pll_write;
 906
 907         adev->mode_info.atom_context = amdgpu_atom_parse(atom_card_info, adev->bios);
 908         if (!adev->mode_info.atom_context) {
 909                 amdgpu_atombios_fini(adev);
 910                 return -ENOMEM;
 911         }
 912
 913         mutex_init(&adev->mode_info.atom_context->mutex);
 914         amdgpu_atombios_scratch_regs_init(adev);
 915         amdgpu_atom_allocate_fb_scratch(adev->mode_info.atom_context);
 916         return 0;
 917 }
 918
 919 /* if we get transitioned to only one device, take VGA back */
 920 /**
 921  * amdgpu_vga_set_decode - enable/disable vga decode
 922  *
 923  * @cookie: amdgpu_device pointer
 924  * @state: enable/disable vga decode
 925  *
 926  * Enable/disable vga decode (all asics).
 927  * Returns VGA resource flags.
 928  */
 929 static unsigned int amdgpu_vga_set_decode(void *cookie, bool state)
 930 {
 931         struct amdgpu_device *adev = cookie;
 932         amdgpu_asic_set_vga_state(adev, state);
 933         if (state)
 934                 return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM |
 935                        VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
 936         else
 937                 return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
 938 }
 939
 940 /**
 941  * amdgpu_check_pot_argument - check that argument is a power of two
 942  *
 943  * @arg: value to check
 944  *
 945  * Validates that a certain argument is a power of two (all asics).
 946  * Returns true if argument is valid.
 947  */
 948 static bool amdgpu_check_pot_argument(int arg)
 949 {
 950         return (arg & (arg - 1)) == 0;
 951 }
 952
 953 /**
 954  * amdgpu_check_arguments - validate module params
 955  *
 956  * @adev: amdgpu_device pointer
 957  *
 958  * Validates certain module parameters and updates
 959  * the associated values used by the driver (all asics).
 960  */
 961 static void amdgpu_check_arguments(struct amdgpu_device *adev)
 962 {
 963         if (amdgpu_sched_jobs < 4) {
 964                 dev_warn(adev->dev, "sched jobs (%d) must be at least 4\n",
 965                          amdgpu_sched_jobs);
 966                 amdgpu_sched_jobs = 4;
 967         } else if (!amdgpu_check_pot_argument(amdgpu_sched_jobs)){
 968                 dev_warn(adev->dev, "sched jobs (%d) must be a power of 2\n",
 969                          amdgpu_sched_jobs);
 970                 amdgpu_sched_jobs = roundup_pow_of_two(amdgpu_sched_jobs);
 971         }
 972
 973         if (amdgpu_gart_size != -1) {
 974                 /* gtt size must be greater or equal to 32M */
 975                 if (amdgpu_gart_size < 32) {
 976                         dev_warn(adev->dev, "gart size (%d) too small\n",
 977                                  amdgpu_gart_size);
 978                         amdgpu_gart_size = -1;
 979                 }
 980         }
 981
 982         if (!amdgpu_check_pot_argument(amdgpu_vm_size)) {
 983                 dev_warn(adev->dev, "VM size (%d) must be a power of 2\n",
 984                          amdgpu_vm_size);
 985                 amdgpu_vm_size = 8;
 986         }
 987
 988         if (amdgpu_vm_size < 1) {
 989                 dev_warn(adev->dev, "VM size (%d) too small, min is 1GB\n",
 990                          amdgpu_vm_size);
 991                 amdgpu_vm_size = 8;
 992         }
 993
 994         /*
 995          * Max GPUVM size for Cayman, SI and CI are 40 bits.
 996          */
 997         if (amdgpu_vm_size > 1024) {
 998                 dev_warn(adev->dev, "VM size (%d) too large, max is 1TB\n",
 999                          amdgpu_vm_size);
1000                 amdgpu_vm_size = 8;
1001         }
1002
1003         /* defines number of bits in page table versus page directory,
1004          * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
1005          * page table and the remaining bits are in the page directory */
1006         if (amdgpu_vm_block_size == -1) {
1007
1008                 /* Total bits covered by PD + PTs */
1009                 unsigned bits = ilog2(amdgpu_vm_size) + 18;
1010
1011                 /* Make sure the PD is 4K in size up to 8GB address space.
1012                    Above that split equal between PD and PTs */
1013                 if (amdgpu_vm_size <= 8)
1014                         amdgpu_vm_block_size = bits - 9;
1015                 else
1016                         amdgpu_vm_block_size = (bits + 3) / 2;
1017
1018         } else if (amdgpu_vm_block_size < 9) {
1019                 dev_warn(adev->dev, "VM page table size (%d) too small\n",
1020                          amdgpu_vm_block_size);
1021                 amdgpu_vm_block_size = 9;
1022         }
1023
1024         if (amdgpu_vm_block_size > 24 ||
1025             (amdgpu_vm_size * 1024) < (1ull << amdgpu_vm_block_size)) {
1026                 dev_warn(adev->dev, "VM page table size (%d) too large\n",
1027                          amdgpu_vm_block_size);
1028                 amdgpu_vm_block_size = 9;
1029         }
1030
1031         if (amdgpu_vram_page_split != -1 && (amdgpu_vram_page_split < 16 ||
1032             !amdgpu_check_pot_argument(amdgpu_vram_page_split))) {
1033                 dev_warn(adev->dev, "invalid VRAM page split (%d)\n",
1034                          amdgpu_vram_page_split);
1035                 amdgpu_vram_page_split = 1024;
1036         }
1037 }
1038
1039 /**
1040  * amdgpu_switcheroo_set_state - set switcheroo state
1041  *
1042  * @pdev: pci dev pointer
1043  * @state: vga_switcheroo state
1044  *
1045  * Callback for the switcheroo driver.  Suspends or resumes the
1046  * the asics before or after it is powered up using ACPI methods.
1047  */
1048 static void amdgpu_switcheroo_set_state(struct pci_dev *pdev, enum vga_switcheroo_state state)
1049 {
1050         struct drm_device *dev = pci_get_drvdata(pdev);
1051
1052         if (amdgpu_device_is_px(dev) && state == VGA_SWITCHEROO_OFF)
1053                 return;
1054
1055         if (state == VGA_SWITCHEROO_ON) {
1056                 unsigned d3_delay = dev->pdev->d3_delay;
1057
1058                 printk(KERN_INFO "amdgpu: switched on\n");
1059                 /* don't suspend or resume card normally */
1060                 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1061
1062                 amdgpu_device_resume(dev, true, true);
1063
1064                 dev->pdev->d3_delay = d3_delay;
1065
1066                 dev->switch_power_state = DRM_SWITCH_POWER_ON;
1067                 drm_kms_helper_poll_enable(dev);
1068         } else {
1069                 printk(KERN_INFO "amdgpu: switched off\n");
1070                 drm_kms_helper_poll_disable(dev);
1071                 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1072                 amdgpu_device_suspend(dev, true, true);
1073                 dev->switch_power_state = DRM_SWITCH_POWER_OFF;
1074         }
1075 }
1076
1077 /**
1078  * amdgpu_switcheroo_can_switch - see if switcheroo state can change
1079  *
1080  * @pdev: pci dev pointer
1081  *
1082  * Callback for the switcheroo driver.  Check of the switcheroo
1083  * state can be changed.
1084  * Returns true if the state can be changed, false if not.
1085  */
1086 static bool amdgpu_switcheroo_can_switch(struct pci_dev *pdev)
1087 {
1088         struct drm_device *dev = pci_get_drvdata(pdev);
1089
1090         /*
1091         * FIXME: open_count is protected by drm_global_mutex but that would lead to
1092         * locking inversion with the driver load path. And the access here is
1093         * completely racy anyway. So don't bother with locking for now.
1094         */
1095         return dev->open_count == 0;
1096 }
1097
1098 static const struct vga_switcheroo_client_ops amdgpu_switcheroo_ops = {
1099         .set_gpu_state = amdgpu_switcheroo_set_state,
1100         .reprobe = NULL,
1101         .can_switch = amdgpu_switcheroo_can_switch,
1102 };
1103
1104 int amdgpu_set_clockgating_state(struct amdgpu_device *adev,
1105                                   enum amd_ip_block_type block_type,
1106                                   enum amd_clockgating_state state)
1107 {
1108         int i, r = 0;
1109
1110         for (i = 0; i < adev->num_ip_blocks; i++) {
1111                 if (!adev->ip_blocks[i].status.valid)
1112                         continue;
1113                 if (adev->ip_blocks[i].version->type == block_type) {
1114                         r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
1115                                                                                      state);
1116                         if (r)
1117                                 return r;
1118                         break;
1119                 }
1120         }
1121         return r;
1122 }
1123
1124 int amdgpu_set_powergating_state(struct amdgpu_device *adev,
1125                                   enum amd_ip_block_type block_type,
1126                                   enum amd_powergating_state state)
1127 {
1128         int i, r = 0;
1129
1130         for (i = 0; i < adev->num_ip_blocks; i++) {
1131                 if (!adev->ip_blocks[i].status.valid)
1132                         continue;
1133                 if (adev->ip_blocks[i].version->type == block_type) {
1134                         r = adev->ip_blocks[i].version->funcs->set_powergating_state((void *)adev,
1135                                                                                      state);
1136                         if (r)
1137                                 return r;
1138                         break;
1139                 }
1140         }
1141         return r;
1142 }
1143
1144 void amdgpu_get_clockgating_state(struct amdgpu_device *adev, u32 *flags)
1145 {
1146         int i;
1147
1148         for (i = 0; i < adev->num_ip_blocks; i++) {
1149                 if (!adev->ip_blocks[i].status.valid)
1150                         continue;
1151                 if (adev->ip_blocks[i].version->funcs->get_clockgating_state)
1152                         adev->ip_blocks[i].version->funcs->get_clockgating_state((void *)adev, flags);
1153         }
1154 }
1155
1156 int amdgpu_wait_for_idle(struct amdgpu_device *adev,
1157                          enum amd_ip_block_type block_type)
1158 {
1159         int i, r;
1160
1161         for (i = 0; i < adev->num_ip_blocks; i++) {
1162                 if (!adev->ip_blocks[i].status.valid)
1163                         continue;
1164                 if (adev->ip_blocks[i].version->type == block_type) {
1165                         r = adev->ip_blocks[i].version->funcs->wait_for_idle((void *)adev);
1166                         if (r)
1167                                 return r;
1168                         break;
1169                 }
1170         }
1171         return 0;
1172
1173 }
1174
1175 bool amdgpu_is_idle(struct amdgpu_device *adev,
1176                     enum amd_ip_block_type block_type)
1177 {
1178         int i;
1179
1180         for (i = 0; i < adev->num_ip_blocks; i++) {
1181                 if (!adev->ip_blocks[i].status.valid)
1182                         continue;
1183                 if (adev->ip_blocks[i].version->type == block_type)
1184                         return adev->ip_blocks[i].version->funcs->is_idle((void *)adev);
1185         }
1186         return true;
1187
1188 }
1189
1190 struct amdgpu_ip_block * amdgpu_get_ip_block(struct amdgpu_device *adev,
1191                                              enum amd_ip_block_type type)
1192 {
1193         int i;
1194
1195         for (i = 0; i < adev->num_ip_blocks; i++)
1196                 if (adev->ip_blocks[i].version->type == type)
1197                         return &adev->ip_blocks[i];
1198
1199         return NULL;
1200 }
1201
1202 /**
1203  * amdgpu_ip_block_version_cmp
1204  *
1205  * @adev: amdgpu_device pointer
1206  * @type: enum amd_ip_block_type
1207  * @major: major version
1208  * @minor: minor version
1209  *
1210  * return 0 if equal or greater
1211  * return 1 if smaller or the ip_block doesn't exist
1212  */
1213 int amdgpu_ip_block_version_cmp(struct amdgpu_device *adev,
1214                                 enum amd_ip_block_type type,
1215                                 u32 major, u32 minor)
1216 {
1217         struct amdgpu_ip_block *ip_block = amdgpu_get_ip_block(adev, type);
1218
1219         if (ip_block && ((ip_block->version->major > major) ||
1220                         ((ip_block->version->major == major) &&
1221                         (ip_block->version->minor >= minor))))
1222                 return 0;
1223
1224         return 1;
1225 }
1226
1227 /**
1228  * amdgpu_ip_block_add
1229  *
1230  * @adev: amdgpu_device pointer
1231  * @ip_block_version: pointer to the IP to add
1232  *
1233  * Adds the IP block driver information to the collection of IPs
1234  * on the asic.
1235  */
1236 int amdgpu_ip_block_add(struct amdgpu_device *adev,
1237                         const struct amdgpu_ip_block_version *ip_block_version)
1238 {
1239         if (!ip_block_version)
1240                 return -EINVAL;
1241
1242         adev->ip_blocks[adev->num_ip_blocks++].version = ip_block_version;
1243
1244         return 0;
1245 }
1246
1247 static void amdgpu_device_enable_virtual_display(struct amdgpu_device *adev)
1248 {
1249         adev->enable_virtual_display = false;
1250
1251         if (amdgpu_virtual_display) {
1252                 struct drm_device *ddev = adev->ddev;
1253                 const char *pci_address_name = pci_name(ddev->pdev);
1254                 char *pciaddstr, *pciaddstr_tmp, *pciaddname_tmp, *pciaddname;
1255
1256                 pciaddstr = kstrdup(amdgpu_virtual_display, GFP_KERNEL);
1257                 pciaddstr_tmp = pciaddstr;
1258                 while ((pciaddname_tmp = strsep(&pciaddstr_tmp, ";"))) {
1259                         pciaddname = strsep(&pciaddname_tmp, ",");
1260                         if (!strcmp("all", pciaddname)
1261                             || !strcmp(pci_address_name, pciaddname)) {
1262                                 long num_crtc;
1263                                 int res = -1;
1264
1265                                 adev->enable_virtual_display = true;
1266
1267                                 if (pciaddname_tmp)
1268                                         res = kstrtol(pciaddname_tmp, 10,
1269                                                       &num_crtc);
1270
1271                                 if (!res) {
1272                                         if (num_crtc < 1)
1273                                                 num_crtc = 1;
1274                                         if (num_crtc > 6)
1275                                                 num_crtc = 6;
1276                                         adev->mode_info.num_crtc = num_crtc;
1277                                 } else {
1278                                         adev->mode_info.num_crtc = 1;
1279                                 }
1280                                 break;
1281                         }
1282                 }
1283
1284                 DRM_INFO("virtual display string:%s, %s:virtual_display:%d, num_crtc:%d\n",
1285                          amdgpu_virtual_display, pci_address_name,
1286                          adev->enable_virtual_display, adev->mode_info.num_crtc);
1287
1288                 kfree(pciaddstr);
1289         }
1290 }
1291
1292 static int amdgpu_early_init(struct amdgpu_device *adev)
1293 {
1294         int i, r;
1295
1296         amdgpu_device_enable_virtual_display(adev);
1297
1298         switch (adev->asic_type) {
1299         case CHIP_TOPAZ:
1300         case CHIP_TONGA:
1301         case CHIP_FIJI:
1302         case CHIP_POLARIS11:
1303         case CHIP_POLARIS10:
1304         case CHIP_POLARIS12:
1305         case CHIP_CARRIZO:
1306         case CHIP_STONEY:
1307                 if (adev->asic_type == CHIP_CARRIZO || adev->asic_type == CHIP_STONEY)
1308                         adev->family = AMDGPU_FAMILY_CZ;
1309                 else
1310                         adev->family = AMDGPU_FAMILY_VI;
1311
1312                 r = vi_set_ip_blocks(adev);
1313                 if (r)
1314                         return r;
1315                 break;
1316 #ifdef CONFIG_DRM_AMDGPU_SI
1317         case CHIP_VERDE:
1318         case CHIP_TAHITI:
1319         case CHIP_PITCAIRN:
1320         case CHIP_OLAND:
1321         case CHIP_HAINAN:
1322                 adev->family = AMDGPU_FAMILY_SI;
1323                 r = si_set_ip_blocks(adev);
1324                 if (r)
1325                         return r;
1326                 break;
1327 #endif
1328 #ifdef CONFIG_DRM_AMDGPU_CIK
1329         case CHIP_BONAIRE:
1330         case CHIP_HAWAII:
1331         case CHIP_KAVERI:
1332         case CHIP_KABINI:
1333         case CHIP_MULLINS:
1334                 if ((adev->asic_type == CHIP_BONAIRE) || (adev->asic_type == CHIP_HAWAII))
1335                         adev->family = AMDGPU_FAMILY_CI;
1336                 else
1337                         adev->family = AMDGPU_FAMILY_KV;
1338
1339                 r = cik_set_ip_blocks(adev);
1340                 if (r)
1341                         return r;
1342                 break;
1343 #endif
1344         default:
1345                 /* FIXME: not supported yet */
1346                 return -EINVAL;
1347         }
1348
1349         if (amdgpu_sriov_vf(adev)) {
1350                 r = amdgpu_virt_request_full_gpu(adev, true);
1351                 if (r)
1352                         return r;
1353         }
1354
1355         for (i = 0; i < adev->num_ip_blocks; i++) {
1356                 if ((amdgpu_ip_block_mask & (1 << i)) == 0) {
1357                         DRM_ERROR("disabled ip block: %d\n", i);
1358                         adev->ip_blocks[i].status.valid = false;
1359                 } else {
1360                         if (adev->ip_blocks[i].version->funcs->early_init) {
1361                                 r = adev->ip_blocks[i].version->funcs->early_init((void *)adev);
1362                                 if (r == -ENOENT) {
1363                                         adev->ip_blocks[i].status.valid = false;
1364                                 } else if (r) {
1365                                         DRM_ERROR("early_init of IP block <%s> failed %d\n",
1366                                                   adev->ip_blocks[i].version->funcs->name, r);
1367                                         return r;
1368                                 } else {
1369                                         adev->ip_blocks[i].status.valid = true;
1370                                 }
1371                         } else {
1372                                 adev->ip_blocks[i].status.valid = true;
1373                         }
1374                 }
1375         }
1376
1377         adev->cg_flags &= amdgpu_cg_mask;
1378         adev->pg_flags &= amdgpu_pg_mask;
1379
1380         return 0;
1381 }
1382
1383 static int amdgpu_init(struct amdgpu_device *adev)
1384 {
1385         int i, r;
1386
1387         for (i = 0; i < adev->num_ip_blocks; i++) {
1388                 if (!adev->ip_blocks[i].status.valid)
1389                         continue;
1390                 r = adev->ip_blocks[i].version->funcs->sw_init((void *)adev);
1391                 if (r) {
1392                         DRM_ERROR("sw_init of IP block <%s> failed %d\n",
1393                                   adev->ip_blocks[i].version->funcs->name, r);
1394                         return r;
1395                 }
1396                 adev->ip_blocks[i].status.sw = true;
1397                 /* need to do gmc hw init early so we can allocate gpu mem */
1398                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
1399                         r = amdgpu_vram_scratch_init(adev);
1400                         if (r) {
1401                                 DRM_ERROR("amdgpu_vram_scratch_init failed %d\n", r);
1402                                 return r;
1403                         }
1404                         r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev);
1405                         if (r) {
1406                                 DRM_ERROR("hw_init %d failed %d\n", i, r);
1407                                 return r;
1408                         }
1409                         r = amdgpu_wb_init(adev);
1410                         if (r) {
1411                                 DRM_ERROR("amdgpu_wb_init failed %d\n", r);
1412                                 return r;
1413                         }
1414                         adev->ip_blocks[i].status.hw = true;
1415
1416                         /* right after GMC hw init, we create CSA */
1417                         if (amdgpu_sriov_vf(adev)) {
1418                                 r = amdgpu_allocate_static_csa(adev);
1419                                 if (r) {
1420                                         DRM_ERROR("allocate CSA failed %d\n", r);
1421                                         return r;
1422                                 }
1423                         }
1424                 }
1425         }
1426
1427         for (i = 0; i < adev->num_ip_blocks; i++) {
1428                 if (!adev->ip_blocks[i].status.sw)
1429                         continue;
1430                 /* gmc hw init is done early */
1431                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC)
1432                         continue;
1433                 r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev);
1434                 if (r) {
1435                         DRM_ERROR("hw_init of IP block <%s> failed %d\n",
1436                                   adev->ip_blocks[i].version->funcs->name, r);
1437                         return r;
1438                 }
1439                 adev->ip_blocks[i].status.hw = true;
1440         }
1441
1442         return 0;
1443 }
1444
1445 static int amdgpu_late_init(struct amdgpu_device *adev)
1446 {
1447         int i = 0, r;
1448
1449         for (i = 0; i < adev->num_ip_blocks; i++) {
1450                 if (!adev->ip_blocks[i].status.valid)
1451                         continue;
1452                 if (adev->ip_blocks[i].version->funcs->late_init) {
1453                         r = adev->ip_blocks[i].version->funcs->late_init((void *)adev);
1454                         if (r) {
1455                                 DRM_ERROR("late_init of IP block <%s> failed %d\n",
1456                                           adev->ip_blocks[i].version->funcs->name, r);
1457                                 return r;
1458                         }
1459                         adev->ip_blocks[i].status.late_initialized = true;
1460                 }
1461                 /* skip CG for VCE/UVD, it's handled specially */
1462                 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
1463                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE) {
1464                         /* enable clockgating to save power */
1465                         r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
1466                                                                                      AMD_CG_STATE_GATE);
1467                         if (r) {
1468                                 DRM_ERROR("set_clockgating_state(gate) of IP block <%s> failed %d\n",
1469                                           adev->ip_blocks[i].version->funcs->name, r);
1470                                 return r;
1471                         }
1472                 }
1473         }
1474
1475         return 0;
1476 }
1477
1478 static int amdgpu_fini(struct amdgpu_device *adev)
1479 {
1480         int i, r;
1481
1482         /* need to disable SMC first */
1483         for (i = 0; i < adev->num_ip_blocks; i++) {
1484                 if (!adev->ip_blocks[i].status.hw)
1485                         continue;
1486                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
1487                         /* ungate blocks before hw fini so that we can shutdown the blocks safely */
1488                         r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
1489                                                                                      AMD_CG_STATE_UNGATE);
1490                         if (r) {
1491                                 DRM_ERROR("set_clockgating_state(ungate) of IP block <%s> failed %d\n",
1492                                           adev->ip_blocks[i].version->funcs->name, r);
1493                                 return r;
1494                         }
1495                         r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
1496                         /* XXX handle errors */
1497                         if (r) {
1498                                 DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
1499                                           adev->ip_blocks[i].version->funcs->name, r);
1500                         }
1501                         adev->ip_blocks[i].status.hw = false;
1502                         break;
1503                 }
1504         }
1505
1506         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
1507                 if (!adev->ip_blocks[i].status.hw)
1508                         continue;
1509                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
1510                         amdgpu_wb_fini(adev);
1511                         amdgpu_vram_scratch_fini(adev);
1512                 }
1513
1514                 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
1515                         adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE) {
1516                         /* ungate blocks before hw fini so that we can shutdown the blocks safely */
1517                         r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
1518                                                                                      AMD_CG_STATE_UNGATE);
1519                         if (r) {
1520                                 DRM_ERROR("set_clockgating_state(ungate) of IP block <%s> failed %d\n",
1521                                           adev->ip_blocks[i].version->funcs->name, r);
1522                                 return r;
1523                         }
1524                 }
1525
1526                 r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
1527                 /* XXX handle errors */
1528                 if (r) {
1529                         DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
1530                                   adev->ip_blocks[i].version->funcs->name, r);
1531                 }
1532
1533                 adev->ip_blocks[i].status.hw = false;
1534         }
1535
1536         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
1537                 if (!adev->ip_blocks[i].status.sw)
1538                         continue;
1539                 r = adev->ip_blocks[i].version->funcs->sw_fini((void *)adev);
1540                 /* XXX handle errors */
1541                 if (r) {
1542                         DRM_DEBUG("sw_fini of IP block <%s> failed %d\n",
1543                                   adev->ip_blocks[i].version->funcs->name, r);
1544                 }
1545                 adev->ip_blocks[i].status.sw = false;
1546                 adev->ip_blocks[i].status.valid = false;
1547         }
1548
1549         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
1550                 if (!adev->ip_blocks[i].status.late_initialized)
1551                         continue;
1552                 if (adev->ip_blocks[i].version->funcs->late_fini)
1553                         adev->ip_blocks[i].version->funcs->late_fini((void *)adev);
1554                 adev->ip_blocks[i].status.late_initialized = false;
1555         }
1556
1557         if (amdgpu_sriov_vf(adev)) {
1558                 amdgpu_bo_free_kernel(&adev->virt.csa_obj, &adev->virt.csa_vmid0_addr, NULL);
1559                 amdgpu_virt_release_full_gpu(adev, false);
1560         }
1561
1562         return 0;
1563 }
1564
1565 int amdgpu_suspend(struct amdgpu_device *adev)
1566 {
1567         int i, r;
1568
1569         if (amdgpu_sriov_vf(adev))
1570                 amdgpu_virt_request_full_gpu(adev, false);
1571
1572         /* ungate SMC block first */
1573         r = amdgpu_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_SMC,
1574                                          AMD_CG_STATE_UNGATE);
1575         if (r) {
1576                 DRM_ERROR("set_clockgating_state(ungate) SMC failed %d\n",r);
1577         }
1578
1579         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
1580                 if (!adev->ip_blocks[i].status.valid)
1581                         continue;
1582                 /* ungate blocks so that suspend can properly shut them down */
1583                 if (i != AMD_IP_BLOCK_TYPE_SMC) {
1584                         r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
1585                                                                                      AMD_CG_STATE_UNGATE);
1586                         if (r) {
1587                                 DRM_ERROR("set_clockgating_state(ungate) of IP block <%s> failed %d\n",
1588                                           adev->ip_blocks[i].version->funcs->name, r);
1589                         }
1590                 }
1591                 /* XXX handle errors */
1592                 r = adev->ip_blocks[i].version->funcs->suspend(adev);
1593                 /* XXX handle errors */
1594                 if (r) {
1595                         DRM_ERROR("suspend of IP block <%s> failed %d\n",
1596                                   adev->ip_blocks[i].version->funcs->name, r);
1597                 }
1598         }
1599
1600         if (amdgpu_sriov_vf(adev))
1601                 amdgpu_virt_release_full_gpu(adev, false);
1602
1603         return 0;
1604 }
1605
1606 static int amdgpu_resume(struct amdgpu_device *adev)
1607 {
1608         int i, r;
1609
1610         for (i = 0; i < adev->num_ip_blocks; i++) {
1611                 if (!adev->ip_blocks[i].status.valid)
1612                         continue;
1613                 r = adev->ip_blocks[i].version->funcs->resume(adev);
1614                 if (r) {
1615                         DRM_ERROR("resume of IP block <%s> failed %d\n",
1616                                   adev->ip_blocks[i].version->funcs->name, r);
1617                         return r;
1618                 }
1619         }
1620
1621         return 0;
1622 }
1623
1624 static void amdgpu_device_detect_sriov_bios(struct amdgpu_device *adev)
1625 {
1626         if (amdgpu_atombios_has_gpu_virtualization_table(adev))
1627                 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
1628 }
1629
1630 /**
1631  * amdgpu_device_init - initialize the driver
1632  *
1633  * @adev: amdgpu_device pointer
1634  * @pdev: drm dev pointer
1635  * @pdev: pci dev pointer
1636  * @flags: driver flags
1637  *
1638  * Initializes the driver info and hw (all asics).
1639  * Returns 0 for success or an error on failure.
1640  * Called at driver startup.
1641  */
1642 int amdgpu_device_init(struct amdgpu_device *adev,
1643                        struct drm_device *ddev,
1644                        struct pci_dev *pdev,
1645                        uint32_t flags)
1646 {
1647         int r, i;
1648         bool runtime = false;
1649         u32 max_MBps;
1650
1651         adev->shutdown = false;
1652         adev->dev = &pdev->dev;
1653         adev->ddev = ddev;
1654         adev->pdev = pdev;
1655         adev->flags = flags;
1656         adev->asic_type = flags & AMD_ASIC_MASK;
1657         adev->usec_timeout = AMDGPU_MAX_USEC_TIMEOUT;
1658         adev->mc.gtt_size = 512 * 1024 * 1024;
1659         adev->accel_working = false;
1660         adev->num_rings = 0;
1661         adev->mman.buffer_funcs = NULL;
1662         adev->mman.buffer_funcs_ring = NULL;
1663         adev->vm_manager.vm_pte_funcs = NULL;
1664         adev->vm_manager.vm_pte_num_rings = 0;
1665         adev->gart.gart_funcs = NULL;
1666         adev->fence_context = dma_fence_context_alloc(AMDGPU_MAX_RINGS);
1667
1668         adev->smc_rreg = &amdgpu_invalid_rreg;
1669         adev->smc_wreg = &amdgpu_invalid_wreg;
1670         adev->pcie_rreg = &amdgpu_invalid_rreg;
1671         adev->pcie_wreg = &amdgpu_invalid_wreg;
1672         adev->pciep_rreg = &amdgpu_invalid_rreg;
1673         adev->pciep_wreg = &amdgpu_invalid_wreg;
1674         adev->uvd_ctx_rreg = &amdgpu_invalid_rreg;
1675         adev->uvd_ctx_wreg = &amdgpu_invalid_wreg;
1676         adev->didt_rreg = &amdgpu_invalid_rreg;
1677         adev->didt_wreg = &amdgpu_invalid_wreg;
1678         adev->gc_cac_rreg = &amdgpu_invalid_rreg;
1679         adev->gc_cac_wreg = &amdgpu_invalid_wreg;
1680         adev->audio_endpt_rreg = &amdgpu_block_invalid_rreg;
1681         adev->audio_endpt_wreg = &amdgpu_block_invalid_wreg;
1682
1683
1684         DRM_INFO("initializing kernel modesetting (%s 0x%04X:0x%04X 0x%04X:0x%04X 0x%02X).\n",
1685                  amdgpu_asic_name[adev->asic_type], pdev->vendor, pdev->device,
1686                  pdev->subsystem_vendor, pdev->subsystem_device, pdev->revision);
1687
1688         /* mutex initialization are all done here so we
1689          * can recall function without having locking issues */
1690         mutex_init(&adev->vm_manager.lock);
1691         atomic_set(&adev->irq.ih.lock, 0);
1692         mutex_init(&adev->pm.mutex);
1693         mutex_init(&adev->gfx.gpu_clock_mutex);
1694         mutex_init(&adev->srbm_mutex);
1695         mutex_init(&adev->grbm_idx_mutex);
1696         mutex_init(&adev->mn_lock);
1697         hash_init(adev->mn_hash);
1698
1699         amdgpu_check_arguments(adev);
1700
1701         /* Registers mapping */
1702         /* TODO: block userspace mapping of io register */
1703         spin_lock_init(&adev->mmio_idx_lock);
1704         spin_lock_init(&adev->smc_idx_lock);
1705         spin_lock_init(&adev->pcie_idx_lock);
1706         spin_lock_init(&adev->uvd_ctx_idx_lock);
1707         spin_lock_init(&adev->didt_idx_lock);
1708         spin_lock_init(&adev->gc_cac_idx_lock);
1709         spin_lock_init(&adev->audio_endpt_idx_lock);
1710         spin_lock_init(&adev->mm_stats.lock);
1711
1712         INIT_LIST_HEAD(&adev->shadow_list);
1713         mutex_init(&adev->shadow_list_lock);
1714
1715         INIT_LIST_HEAD(&adev->gtt_list);
1716         spin_lock_init(&adev->gtt_list_lock);
1717
1718         if (adev->asic_type >= CHIP_BONAIRE) {
1719                 adev->rmmio_base = pci_resource_start(adev->pdev, 5);
1720                 adev->rmmio_size = pci_resource_len(adev->pdev, 5);
1721         } else {
1722                 adev->rmmio_base = pci_resource_start(adev->pdev, 2);
1723                 adev->rmmio_size = pci_resource_len(adev->pdev, 2);
1724         }
1725
1726         adev->rmmio = ioremap(adev->rmmio_base, adev->rmmio_size);
1727         if (adev->rmmio == NULL) {
1728                 return -ENOMEM;
1729         }
1730         DRM_INFO("register mmio base: 0x%08X\n", (uint32_t)adev->rmmio_base);
1731         DRM_INFO("register mmio size: %u\n", (unsigned)adev->rmmio_size);
1732
1733         if (adev->asic_type >= CHIP_BONAIRE)
1734                 /* doorbell bar mapping */
1735                 amdgpu_doorbell_init(adev);
1736
1737         /* io port mapping */
1738         for (i = 0; i < DEVICE_COUNT_RESOURCE; i++) {
1739                 if (pci_resource_flags(adev->pdev, i) & IORESOURCE_IO) {
1740                         adev->rio_mem_size = pci_resource_len(adev->pdev, i);
1741                         adev->rio_mem = pci_iomap(adev->pdev, i, adev->rio_mem_size);
1742                         break;
1743                 }
1744         }
1745         if (adev->rio_mem == NULL)
1746                 DRM_INFO("PCI I/O BAR is not found.\n");
1747
1748         /* early init functions */
1749         r = amdgpu_early_init(adev);
1750         if (r)
1751                 return r;
1752
1753         /* if we have > 1 VGA cards, then disable the amdgpu VGA resources */
1754         /* this will fail for cards that aren't VGA class devices, just
1755          * ignore it */
1756         vga_client_register(adev->pdev, adev, NULL, amdgpu_vga_set_decode);
1757
1758         if (amdgpu_runtime_pm == 1)
1759                 runtime = true;
1760         if (amdgpu_device_is_px(ddev))
1761                 runtime = true;
1762         vga_switcheroo_register_client(adev->pdev, &amdgpu_switcheroo_ops, runtime);
1763         if (runtime)
1764                 vga_switcheroo_init_domain_pm_ops(adev->dev, &adev->vga_pm_domain);
1765
1766         /* Read BIOS */
1767         if (!amdgpu_get_bios(adev)) {
1768                 r = -EINVAL;
1769                 goto failed;
1770         }
1771
1772         r = amdgpu_atombios_init(adev);
1773         if (r) {
1774                 dev_err(adev->dev, "amdgpu_atombios_init failed\n");
1775                 goto failed;
1776         }
1777
1778         /* detect if we are with an SRIOV vbios */
1779         amdgpu_device_detect_sriov_bios(adev);
1780
1781         /* Post card if necessary */
1782         if (amdgpu_vpost_needed(adev)) {
1783                 if (!adev->bios) {
1784                         dev_err(adev->dev, "no vBIOS found\n");
1785                         r = -EINVAL;
1786                         goto failed;
1787                 }
1788                 DRM_INFO("GPU posting now...\n");
1789                 r = amdgpu_atom_asic_init(adev->mode_info.atom_context);
1790                 if (r) {
1791                         dev_err(adev->dev, "gpu post error!\n");
1792                         goto failed;
1793                 }
1794         } else {
1795                 DRM_INFO("GPU post is not needed\n");
1796         }
1797
1798         /* Initialize clocks */
1799         r = amdgpu_atombios_get_clock_info(adev);
1800         if (r) {
1801                 dev_err(adev->dev, "amdgpu_atombios_get_clock_info failed\n");
1802                 goto failed;
1803         }
1804         /* init i2c buses */
1805         amdgpu_atombios_i2c_init(adev);
1806
1807         /* Fence driver */
1808         r = amdgpu_fence_driver_init(adev);
1809         if (r) {
1810                 dev_err(adev->dev, "amdgpu_fence_driver_init failed\n");
1811                 goto failed;
1812         }
1813
1814         /* init the mode config */
1815         drm_mode_config_init(adev->ddev);
1816
1817         r = amdgpu_init(adev);
1818         if (r) {
1819                 dev_err(adev->dev, "amdgpu_init failed\n");
1820                 amdgpu_fini(adev);
1821                 goto failed;
1822         }
1823
1824         adev->accel_working = true;
1825
1826         /* Initialize the buffer migration limit. */
1827         if (amdgpu_moverate >= 0)
1828                 max_MBps = amdgpu_moverate;
1829         else
1830                 max_MBps = 8; /* Allow 8 MB/s. */
1831         /* Get a log2 for easy divisions. */
1832         adev->mm_stats.log2_max_MBps = ilog2(max(1u, max_MBps));
1833
1834         amdgpu_fbdev_init(adev);
1835
1836         r = amdgpu_ib_pool_init(adev);
1837         if (r) {
1838                 dev_err(adev->dev, "IB initialization failed (%d).\n", r);
1839                 goto failed;
1840         }
1841
1842         r = amdgpu_ib_ring_tests(adev);
1843         if (r)
1844                 DRM_ERROR("ib ring test failed (%d).\n", r);
1845
1846         r = amdgpu_gem_debugfs_init(adev);
1847         if (r) {
1848                 DRM_ERROR("registering gem debugfs failed (%d).\n", r);
1849         }
1850
1851         r = amdgpu_debugfs_regs_init(adev);
1852         if (r) {
1853                 DRM_ERROR("registering register debugfs failed (%d).\n", r);
1854         }
1855
1856         r = amdgpu_debugfs_firmware_init(adev);
1857         if (r) {
1858                 DRM_ERROR("registering firmware debugfs failed (%d).\n", r);
1859                 return r;
1860         }
1861
1862         if ((amdgpu_testing & 1)) {
1863                 if (adev->accel_working)
1864                         amdgpu_test_moves(adev);
1865                 else
1866                         DRM_INFO("amdgpu: acceleration disabled, skipping move tests\n");
1867         }
1868         if ((amdgpu_testing & 2)) {
1869                 if (adev->accel_working)
1870                         amdgpu_test_syncing(adev);
1871                 else
1872                         DRM_INFO("amdgpu: acceleration disabled, skipping sync tests\n");
1873         }
1874         if (amdgpu_benchmarking) {
1875                 if (adev->accel_working)
1876                         amdgpu_benchmark(adev, amdgpu_benchmarking);
1877                 else
1878                         DRM_INFO("amdgpu: acceleration disabled, skipping benchmarks\n");
1879         }
1880
1881         /* enable clockgating, etc. after ib tests, etc. since some blocks require
1882          * explicit gating rather than handling it automatically.
1883          */
1884         r = amdgpu_late_init(adev);
1885         if (r) {
1886                 dev_err(adev->dev, "amdgpu_late_init failed\n");
1887                 goto failed;
1888         }
1889
1890         return 0;
1891
1892 failed:
1893         if (runtime)
1894                 vga_switcheroo_fini_domain_pm_ops(adev->dev);
1895         return r;
1896 }
1897
1898 /**
1899  * amdgpu_device_fini - tear down the driver
1900  *
1901  * @adev: amdgpu_device pointer
1902  *
1903  * Tear down the driver info (all asics).
1904  * Called at driver shutdown.
1905  */
1906 void amdgpu_device_fini(struct amdgpu_device *adev)
1907 {
1908         int r;
1909
1910         DRM_INFO("amdgpu: finishing device.\n");
1911         adev->shutdown = true;
1912         drm_crtc_force_disable_all(adev->ddev);
1913         /* evict vram memory */
1914         amdgpu_bo_evict_vram(adev);
1915         amdgpu_ib_pool_fini(adev);
1916         amdgpu_fence_driver_fini(adev);
1917         amdgpu_fbdev_fini(adev);
1918         r = amdgpu_fini(adev);
1919         adev->accel_working = false;
1920         /* free i2c buses */
1921         amdgpu_i2c_fini(adev);
1922         amdgpu_atombios_fini(adev);
1923         kfree(adev->bios);
1924         adev->bios = NULL;
1925         vga_switcheroo_unregister_client(adev->pdev);
1926         if (adev->flags & AMD_IS_PX)
1927                 vga_switcheroo_fini_domain_pm_ops(adev->dev);
1928         vga_client_register(adev->pdev, NULL, NULL, NULL);
1929         if (adev->rio_mem)
1930                 pci_iounmap(adev->pdev, adev->rio_mem);
1931         adev->rio_mem = NULL;
1932         iounmap(adev->rmmio);
1933         adev->rmmio = NULL;
1934         if (adev->asic_type >= CHIP_BONAIRE)
1935                 amdgpu_doorbell_fini(adev);
1936         amdgpu_debugfs_regs_cleanup(adev);
1937 }
1938
1939
1940 /*
1941  * Suspend & resume.
1942  */
1943 /**
1944  * amdgpu_device_suspend - initiate device suspend
1945  *
1946  * @pdev: drm dev pointer
1947  * @state: suspend state
1948  *
1949  * Puts the hw in the suspend state (all asics).
1950  * Returns 0 for success or an error on failure.
1951  * Called at driver suspend.
1952  */
1953 int amdgpu_device_suspend(struct drm_device *dev, bool suspend, bool fbcon)
1954 {
1955         struct amdgpu_device *adev;
1956         struct drm_crtc *crtc;
1957         struct drm_connector *connector;
1958         int r;
1959
1960         if (dev == NULL || dev->dev_private == NULL) {
1961                 return -ENODEV;
1962         }
1963
1964         adev = dev->dev_private;
1965
1966         if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
1967                 return 0;
1968
1969         drm_kms_helper_poll_disable(dev);
1970
1971         /* turn off display hw */
1972         drm_modeset_lock_all(dev);
1973         list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1974                 drm_helper_connector_dpms(connector, DRM_MODE_DPMS_OFF);
1975         }
1976         drm_modeset_unlock_all(dev);
1977
1978         /* unpin the front buffers and cursors */
1979         list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
1980                 struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
1981                 struct amdgpu_framebuffer *rfb = to_amdgpu_framebuffer(crtc->primary->fb);
1982                 struct amdgpu_bo *robj;
1983
1984                 if (amdgpu_crtc->cursor_bo) {
1985                         struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo);
1986                         r = amdgpu_bo_reserve(aobj, false);
1987                         if (r == 0) {
1988                                 amdgpu_bo_unpin(aobj);
1989                                 amdgpu_bo_unreserve(aobj);
1990                         }
1991                 }
1992
1993                 if (rfb == NULL || rfb->obj == NULL) {
1994                         continue;
1995                 }
1996                 robj = gem_to_amdgpu_bo(rfb->obj);
1997                 /* don't unpin kernel fb objects */
1998                 if (!amdgpu_fbdev_robj_is_fb(adev, robj)) {
1999                         r = amdgpu_bo_reserve(robj, false);
2000                         if (r == 0) {
2001                                 amdgpu_bo_unpin(robj);
2002                                 amdgpu_bo_unreserve(robj);
2003                         }
2004                 }
2005         }
2006         /* evict vram memory */
2007         amdgpu_bo_evict_vram(adev);
2008
2009         amdgpu_fence_driver_suspend(adev);
2010
2011         r = amdgpu_suspend(adev);
2012
2013         /* evict remaining vram memory
2014          * This second call to evict vram is to evict the gart page table
2015          * using the CPU.
2016          */
2017         amdgpu_bo_evict_vram(adev);
2018
2019         amdgpu_atombios_scratch_regs_save(adev);
2020         pci_save_state(dev->pdev);
2021         if (suspend) {
2022                 /* Shut down the device */
2023                 pci_disable_device(dev->pdev);
2024                 pci_set_power_state(dev->pdev, PCI_D3hot);
2025         } else {
2026                 r = amdgpu_asic_reset(adev);
2027                 if (r)
2028                         DRM_ERROR("amdgpu asic reset failed\n");
2029         }
2030
2031         if (fbcon) {
2032                 console_lock();
2033                 amdgpu_fbdev_set_suspend(adev, 1);
2034                 console_unlock();
2035         }
2036         return 0;
2037 }
2038
2039 /**
2040  * amdgpu_device_resume - initiate device resume
2041  *
2042  * @pdev: drm dev pointer
2043  *
2044  * Bring the hw back to operating state (all asics).
2045  * Returns 0 for success or an error on failure.
2046  * Called at driver resume.
2047  */
2048 int amdgpu_device_resume(struct drm_device *dev, bool resume, bool fbcon)
2049 {
2050         struct drm_connector *connector;
2051         struct amdgpu_device *adev = dev->dev_private;
2052         struct drm_crtc *crtc;
2053         int r;
2054
2055         if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
2056                 return 0;
2057
2058         if (fbcon)
2059                 console_lock();
2060
2061         if (resume) {
2062                 pci_set_power_state(dev->pdev, PCI_D0);
2063                 pci_restore_state(dev->pdev);
2064                 r = pci_enable_device(dev->pdev);
2065                 if (r) {
2066                         if (fbcon)
2067                                 console_unlock();
2068                         return r;
2069                 }
2070         }
2071         amdgpu_atombios_scratch_regs_restore(adev);
2072
2073         /* post card */
2074         if (!amdgpu_card_posted(adev) || !resume) {
2075                 r = amdgpu_atom_asic_init(adev->mode_info.atom_context);
2076                 if (r)
2077                         DRM_ERROR("amdgpu asic init failed\n");
2078         }
2079
2080         r = amdgpu_resume(adev);
2081         if (r)
2082                 DRM_ERROR("amdgpu_resume failed (%d).\n", r);
2083
2084         amdgpu_fence_driver_resume(adev);
2085
2086         if (resume) {
2087                 r = amdgpu_ib_ring_tests(adev);
2088                 if (r)
2089                         DRM_ERROR("ib ring test failed (%d).\n", r);
2090         }
2091
2092         r = amdgpu_late_init(adev);
2093         if (r)
2094                 return r;
2095
2096         /* pin cursors */
2097         list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
2098                 struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
2099
2100                 if (amdgpu_crtc->cursor_bo) {
2101                         struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo);
2102                         r = amdgpu_bo_reserve(aobj, false);
2103                         if (r == 0) {
2104                                 r = amdgpu_bo_pin(aobj,
2105                                                   AMDGPU_GEM_DOMAIN_VRAM,
2106                                                   &amdgpu_crtc->cursor_addr);
2107                                 if (r != 0)
2108                                         DRM_ERROR("Failed to pin cursor BO (%d)\n", r);
2109                                 amdgpu_bo_unreserve(aobj);
2110                         }
2111                 }
2112         }
2113
2114         /* blat the mode back in */
2115         if (fbcon) {
2116                 drm_helper_resume_force_mode(dev);
2117                 /* turn on display hw */
2118                 drm_modeset_lock_all(dev);
2119                 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
2120                         drm_helper_connector_dpms(connector, DRM_MODE_DPMS_ON);
2121                 }
2122                 drm_modeset_unlock_all(dev);
2123         }
2124
2125         drm_kms_helper_poll_enable(dev);
2126
2127         /*
2128          * Most of the connector probing functions try to acquire runtime pm
2129          * refs to ensure that the GPU is powered on when connector polling is
2130          * performed. Since we're calling this from a runtime PM callback,
2131          * trying to acquire rpm refs will cause us to deadlock.
2132          *
2133          * Since we're guaranteed to be holding the rpm lock, it's safe to
2134          * temporarily disable the rpm helpers so this doesn't deadlock us.
2135          */
2136 #ifdef CONFIG_PM
2137         dev->dev->power.disable_depth++;
2138 #endif
2139         drm_helper_hpd_irq_event(dev);
2140 #ifdef CONFIG_PM
2141         dev->dev->power.disable_depth--;
2142 #endif
2143
2144         if (fbcon) {
2145                 amdgpu_fbdev_set_suspend(adev, 0);
2146                 console_unlock();
2147         }
2148
2149         return 0;
2150 }
2151
2152 static bool amdgpu_check_soft_reset(struct amdgpu_device *adev)
2153 {
2154         int i;
2155         bool asic_hang = false;
2156
2157         for (i = 0; i < adev->num_ip_blocks; i++) {
2158                 if (!adev->ip_blocks[i].status.valid)
2159                         continue;
2160                 if (adev->ip_blocks[i].version->funcs->check_soft_reset)
2161                         adev->ip_blocks[i].status.hang =
2162                                 adev->ip_blocks[i].version->funcs->check_soft_reset(adev);
2163                 if (adev->ip_blocks[i].status.hang) {
2164                         DRM_INFO("IP block:%s is hung!\n", adev->ip_blocks[i].version->funcs->name);
2165                         asic_hang = true;
2166                 }
2167         }
2168         return asic_hang;
2169 }
2170
2171 static int amdgpu_pre_soft_reset(struct amdgpu_device *adev)
2172 {
2173         int i, r = 0;
2174
2175         for (i = 0; i < adev->num_ip_blocks; i++) {
2176                 if (!adev->ip_blocks[i].status.valid)
2177                         continue;
2178                 if (adev->ip_blocks[i].status.hang &&
2179                     adev->ip_blocks[i].version->funcs->pre_soft_reset) {
2180                         r = adev->ip_blocks[i].version->funcs->pre_soft_reset(adev);
2181                         if (r)
2182                                 return r;
2183                 }
2184         }
2185
2186         return 0;
2187 }
2188
2189 static bool amdgpu_need_full_reset(struct amdgpu_device *adev)
2190 {
2191         int i;
2192
2193         for (i = 0; i < adev->num_ip_blocks; i++) {
2194                 if (!adev->ip_blocks[i].status.valid)
2195                         continue;
2196                 if ((adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) ||
2197                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) ||
2198                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_ACP) ||
2199                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE)) {
2200                         if (adev->ip_blocks[i].status.hang) {
2201                                 DRM_INFO("Some block need full reset!\n");
2202                                 return true;
2203                         }
2204                 }
2205         }
2206         return false;
2207 }
2208
2209 static int amdgpu_soft_reset(struct amdgpu_device *adev)
2210 {
2211         int i, r = 0;
2212
2213         for (i = 0; i < adev->num_ip_blocks; i++) {
2214                 if (!adev->ip_blocks[i].status.valid)
2215                         continue;
2216                 if (adev->ip_blocks[i].status.hang &&
2217                     adev->ip_blocks[i].version->funcs->soft_reset) {
2218                         r = adev->ip_blocks[i].version->funcs->soft_reset(adev);
2219                         if (r)
2220                                 return r;
2221                 }
2222         }
2223
2224         return 0;
2225 }
2226
2227 static int amdgpu_post_soft_reset(struct amdgpu_device *adev)
2228 {
2229         int i, r = 0;
2230
2231         for (i = 0; i < adev->num_ip_blocks; i++) {
2232                 if (!adev->ip_blocks[i].status.valid)
2233                         continue;
2234                 if (adev->ip_blocks[i].status.hang &&
2235                     adev->ip_blocks[i].version->funcs->post_soft_reset)
2236                         r = adev->ip_blocks[i].version->funcs->post_soft_reset(adev);
2237                 if (r)
2238                         return r;
2239         }
2240
2241         return 0;
2242 }
2243
2244 bool amdgpu_need_backup(struct amdgpu_device *adev)
2245 {
2246         if (adev->flags & AMD_IS_APU)
2247                 return false;
2248
2249         return amdgpu_lockup_timeout > 0 ? true : false;
2250 }
2251
2252 static int amdgpu_recover_vram_from_shadow(struct amdgpu_device *adev,
2253                                            struct amdgpu_ring *ring,
2254                                            struct amdgpu_bo *bo,
2255                                            struct dma_fence **fence)
2256 {
2257         uint32_t domain;
2258         int r;
2259
2260        if (!bo->shadow)
2261                return 0;
2262
2263        r = amdgpu_bo_reserve(bo, false);
2264        if (r)
2265                return r;
2266        domain = amdgpu_mem_type_to_domain(bo->tbo.mem.mem_type);
2267        /* if bo has been evicted, then no need to recover */
2268        if (domain == AMDGPU_GEM_DOMAIN_VRAM) {
2269                r = amdgpu_bo_restore_from_shadow(adev, ring, bo,
2270                                                  NULL, fence, true);
2271                if (r) {
2272                        DRM_ERROR("recover page table failed!\n");
2273                        goto err;
2274                }
2275        }
2276 err:
2277        amdgpu_bo_unreserve(bo);
2278        return r;
2279 }
2280
2281 /**
2282  * amdgpu_gpu_reset - reset the asic
2283  *
2284  * @adev: amdgpu device pointer
2285  *
2286  * Attempt the reset the GPU if it has hung (all asics).
2287  * Returns 0 for success or an error on failure.
2288  */
2289 int amdgpu_gpu_reset(struct amdgpu_device *adev)
2290 {
2291         int i, r;
2292         int resched;
2293         bool need_full_reset;
2294
2295         if (amdgpu_sriov_vf(adev))
2296                 return 0;
2297
2298         if (!amdgpu_check_soft_reset(adev)) {
2299                 DRM_INFO("No hardware hang detected. Did some blocks stall?\n");
2300                 return 0;
2301         }
2302
2303         atomic_inc(&adev->gpu_reset_counter);
2304
2305         /* block TTM */
2306         resched = ttm_bo_lock_delayed_workqueue(&adev->mman.bdev);
2307
2308         /* block scheduler */
2309         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
2310                 struct amdgpu_ring *ring = adev->rings[i];
2311
2312                 if (!ring)
2313                         continue;
2314                 kthread_park(ring->sched.thread);
2315                 amd_sched_hw_job_reset(&ring->sched);
2316         }
2317         /* after all hw jobs are reset, hw fence is meaningless, so force_completion */
2318         amdgpu_fence_driver_force_completion(adev);
2319
2320         need_full_reset = amdgpu_need_full_reset(adev);
2321
2322         if (!need_full_reset) {
2323                 amdgpu_pre_soft_reset(adev);
2324                 r = amdgpu_soft_reset(adev);
2325                 amdgpu_post_soft_reset(adev);
2326                 if (r || amdgpu_check_soft_reset(adev)) {
2327                         DRM_INFO("soft reset failed, will fallback to full reset!\n");
2328                         need_full_reset = true;
2329                 }
2330         }
2331
2332         if (need_full_reset) {
2333                 r = amdgpu_suspend(adev);
2334
2335 retry:
2336                 /* Disable fb access */
2337                 if (adev->mode_info.num_crtc) {
2338                         struct amdgpu_mode_mc_save save;
2339                         amdgpu_display_stop_mc_access(adev, &save);
2340                         amdgpu_wait_for_idle(adev, AMD_IP_BLOCK_TYPE_GMC);
2341                 }
2342                 amdgpu_atombios_scratch_regs_save(adev);
2343                 r = amdgpu_asic_reset(adev);
2344                 amdgpu_atombios_scratch_regs_restore(adev);
2345                 /* post card */
2346                 amdgpu_atom_asic_init(adev->mode_info.atom_context);
2347
2348                 if (!r) {
2349                         dev_info(adev->dev, "GPU reset succeeded, trying to resume\n");
2350                         r = amdgpu_resume(adev);
2351                 }
2352         }
2353         if (!r) {
2354                 amdgpu_irq_gpu_reset_resume_helper(adev);
2355                 if (need_full_reset && amdgpu_need_backup(adev)) {
2356                         r = amdgpu_ttm_recover_gart(adev);
2357                         if (r)
2358                                 DRM_ERROR("gart recovery failed!!!\n");
2359                 }
2360                 r = amdgpu_ib_ring_tests(adev);
2361                 if (r) {
2362                         dev_err(adev->dev, "ib ring test failed (%d).\n", r);
2363                         r = amdgpu_suspend(adev);
2364                         need_full_reset = true;
2365                         goto retry;
2366                 }
2367                 /**
2368                  * recovery vm page tables, since we cannot depend on VRAM is
2369                  * consistent after gpu full reset.
2370                  */
2371                 if (need_full_reset && amdgpu_need_backup(adev)) {
2372                         struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
2373                         struct amdgpu_bo *bo, *tmp;
2374                         struct dma_fence *fence = NULL, *next = NULL;
2375
2376                         DRM_INFO("recover vram bo from shadow\n");
2377                         mutex_lock(&adev->shadow_list_lock);
2378                         list_for_each_entry_safe(bo, tmp, &adev->shadow_list, shadow_list) {
2379                                 amdgpu_recover_vram_from_shadow(adev, ring, bo, &next);
2380                                 if (fence) {
2381                                         r = dma_fence_wait(fence, false);
2382                                         if (r) {
2383                                                 WARN(r, "recovery from shadow isn't comleted\n");
2384                                                 break;
2385                                         }
2386                                 }
2387
2388                                 dma_fence_put(fence);
2389                                 fence = next;
2390                         }
2391                         mutex_unlock(&adev->shadow_list_lock);
2392                         if (fence) {
2393                                 r = dma_fence_wait(fence, false);
2394                                 if (r)
2395                                         WARN(r, "recovery from shadow isn't comleted\n");
2396                         }
2397                         dma_fence_put(fence);
2398                 }
2399                 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
2400                         struct amdgpu_ring *ring = adev->rings[i];
2401                         if (!ring)
2402                                 continue;
2403
2404                         amd_sched_job_recovery(&ring->sched);
2405                         kthread_unpark(ring->sched.thread);
2406                 }
2407         } else {
2408                 dev_err(adev->dev, "asic resume failed (%d).\n", r);
2409                 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
2410                         if (adev->rings[i]) {
2411                                 kthread_unpark(adev->rings[i]->sched.thread);
2412                         }
2413                 }
2414         }
2415
2416         drm_helper_resume_force_mode(adev->ddev);
2417
2418         ttm_bo_unlock_delayed_workqueue(&adev->mman.bdev, resched);
2419         if (r) {
2420                 /* bad news, how to tell it to userspace ? */
2421                 dev_info(adev->dev, "GPU reset failed\n");
2422         }
2423
2424         return r;
2425 }
2426
2427 void amdgpu_get_pcie_info(struct amdgpu_device *adev)
2428 {
2429         u32 mask;
2430         int ret;
2431
2432         if (amdgpu_pcie_gen_cap)
2433                 adev->pm.pcie_gen_mask = amdgpu_pcie_gen_cap;
2434
2435         if (amdgpu_pcie_lane_cap)
2436                 adev->pm.pcie_mlw_mask = amdgpu_pcie_lane_cap;
2437
2438         /* covers APUs as well */
2439         if (pci_is_root_bus(adev->pdev->bus)) {
2440                 if (adev->pm.pcie_gen_mask == 0)
2441                         adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK;
2442                 if (adev->pm.pcie_mlw_mask == 0)
2443                         adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK;
2444                 return;
2445         }
2446
2447         if (adev->pm.pcie_gen_mask == 0) {
2448                 ret = drm_pcie_get_speed_cap_mask(adev->ddev, &mask);
2449                 if (!ret) {
2450                         adev->pm.pcie_gen_mask = (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
2451                                                   CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
2452                                                   CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
2453
2454                         if (mask & DRM_PCIE_SPEED_25)
2455                                 adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1;
2456                         if (mask & DRM_PCIE_SPEED_50)
2457                                 adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2;
2458                         if (mask & DRM_PCIE_SPEED_80)
2459                                 adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3;
2460                 } else {
2461                         adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK;
2462                 }
2463         }
2464         if (adev->pm.pcie_mlw_mask == 0) {
2465                 ret = drm_pcie_get_max_link_width(adev->ddev, &mask);
2466                 if (!ret) {
2467                         switch (mask) {
2468                         case 32:
2469                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X32 |
2470                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
2471                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
2472                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
2473                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
2474                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
2475                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
2476                                 break;
2477                         case 16:
2478                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
2479                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
2480                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
2481                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
2482                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
2483                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
2484                                 break;
2485                         case 12:
2486                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
2487                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
2488                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
2489                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
2490                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
2491                                 break;
2492                         case 8:
2493                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
2494                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
2495                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
2496                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
2497                                 break;
2498                         case 4:
2499                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
2500                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
2501                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
2502                                 break;
2503                         case 2:
2504                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
2505                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
2506                                 break;
2507                         case 1:
2508                                 adev->pm.pcie_mlw_mask = CAIL_PCIE_LINK_WIDTH_SUPPORT_X1;
2509                                 break;
2510                         default:
2511                                 break;
2512                         }
2513                 } else {
2514                         adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK;
2515                 }
2516         }
2517 }
2518
2519 /*
2520  * Debugfs
2521  */
2522 int amdgpu_debugfs_add_files(struct amdgpu_device *adev,
2523                              const struct drm_info_list *files,
2524                              unsigned nfiles)
2525 {
2526         unsigned i;
2527
2528         for (i = 0; i < adev->debugfs_count; i++) {
2529                 if (adev->debugfs[i].files == files) {
2530                         /* Already registered */
2531                         return 0;
2532                 }
2533         }
2534
2535         i = adev->debugfs_count + 1;
2536         if (i > AMDGPU_DEBUGFS_MAX_COMPONENTS) {
2537                 DRM_ERROR("Reached maximum number of debugfs components.\n");
2538                 DRM_ERROR("Report so we increase "
2539                           "AMDGPU_DEBUGFS_MAX_COMPONENTS.\n");
2540                 return -EINVAL;
2541         }
2542         adev->debugfs[adev->debugfs_count].files = files;
2543         adev->debugfs[adev->debugfs_count].num_files = nfiles;
2544         adev->debugfs_count = i;
2545 #if defined(CONFIG_DEBUG_FS)
2546         drm_debugfs_create_files(files, nfiles,
2547                                  adev->ddev->primary->debugfs_root,
2548                                  adev->ddev->primary);
2549 #endif
2550         return 0;
2551 }
2552
2553 #if defined(CONFIG_DEBUG_FS)
2554
2555 static ssize_t amdgpu_debugfs_regs_read(struct file *f, char __user *buf,
2556                                         size_t size, loff_t *pos)
2557 {
2558         struct amdgpu_device *adev = file_inode(f)->i_private;
2559         ssize_t result = 0;
2560         int r;
2561         bool pm_pg_lock, use_bank;
2562         unsigned instance_bank, sh_bank, se_bank;
2563
2564         if (size & 0x3 || *pos & 0x3)
2565                 return -EINVAL;
2566
2567         /* are we reading registers for which a PG lock is necessary? */
2568         pm_pg_lock = (*pos >> 23) & 1;
2569
2570         if (*pos & (1ULL << 62)) {
2571                 se_bank = (*pos >> 24) & 0x3FF;
2572                 sh_bank = (*pos >> 34) & 0x3FF;
2573                 instance_bank = (*pos >> 44) & 0x3FF;
2574
2575                 if (se_bank == 0x3FF)
2576                         se_bank = 0xFFFFFFFF;
2577                 if (sh_bank == 0x3FF)
2578                         sh_bank = 0xFFFFFFFF;
2579                 if (instance_bank == 0x3FF)
2580                         instance_bank = 0xFFFFFFFF;
2581                 use_bank = 1;
2582         } else {
2583                 use_bank = 0;
2584         }
2585
2586         *pos &= 0x3FFFF;
2587
2588         if (use_bank) {
2589                 if ((sh_bank != 0xFFFFFFFF && sh_bank >= adev->gfx.config.max_sh_per_se) ||
2590                     (se_bank != 0xFFFFFFFF && se_bank >= adev->gfx.config.max_shader_engines))
2591                         return -EINVAL;
2592                 mutex_lock(&adev->grbm_idx_mutex);
2593                 amdgpu_gfx_select_se_sh(adev, se_bank,
2594                                         sh_bank, instance_bank);
2595         }
2596
2597         if (pm_pg_lock)
2598                 mutex_lock(&adev->pm.mutex);
2599
2600         while (size) {
2601                 uint32_t value;
2602
2603                 if (*pos > adev->rmmio_size)
2604                         goto end;
2605
2606                 value = RREG32(*pos >> 2);
2607                 r = put_user(value, (uint32_t *)buf);
2608                 if (r) {
2609                         result = r;
2610                         goto end;
2611                 }
2612
2613                 result += 4;
2614                 buf += 4;
2615                 *pos += 4;
2616                 size -= 4;
2617         }
2618
2619 end:
2620         if (use_bank) {
2621                 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
2622                 mutex_unlock(&adev->grbm_idx_mutex);
2623         }
2624
2625         if (pm_pg_lock)
2626                 mutex_unlock(&adev->pm.mutex);
2627
2628         return result;
2629 }
2630
2631 static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,
2632                                          size_t size, loff_t *pos)
2633 {
2634         struct amdgpu_device *adev = file_inode(f)->i_private;
2635         ssize_t result = 0;
2636         int r;
2637         bool pm_pg_lock, use_bank;
2638         unsigned instance_bank, sh_bank, se_bank;
2639
2640         if (size & 0x3 || *pos & 0x3)
2641                 return -EINVAL;
2642
2643         /* are we reading registers for which a PG lock is necessary? */
2644         pm_pg_lock = (*pos >> 23) & 1;
2645
2646         if (*pos & (1ULL << 62)) {
2647                 se_bank = (*pos >> 24) & 0x3FF;
2648                 sh_bank = (*pos >> 34) & 0x3FF;
2649                 instance_bank = (*pos >> 44) & 0x3FF;
2650
2651                 if (se_bank == 0x3FF)
2652                         se_bank = 0xFFFFFFFF;
2653                 if (sh_bank == 0x3FF)
2654                         sh_bank = 0xFFFFFFFF;
2655                 if (instance_bank == 0x3FF)
2656                         instance_bank = 0xFFFFFFFF;
2657                 use_bank = 1;
2658         } else {
2659                 use_bank = 0;
2660         }
2661
2662         *pos &= 0x3FFFF;
2663
2664         if (use_bank) {
2665                 if ((sh_bank != 0xFFFFFFFF && sh_bank >= adev->gfx.config.max_sh_per_se) ||
2666                     (se_bank != 0xFFFFFFFF && se_bank >= adev->gfx.config.max_shader_engines))
2667                         return -EINVAL;
2668                 mutex_lock(&adev->grbm_idx_mutex);
2669                 amdgpu_gfx_select_se_sh(adev, se_bank,
2670                                         sh_bank, instance_bank);
2671         }
2672
2673         if (pm_pg_lock)
2674                 mutex_lock(&adev->pm.mutex);
2675
2676         while (size) {
2677                 uint32_t value;
2678
2679                 if (*pos > adev->rmmio_size)
2680                         return result;
2681
2682                 r = get_user(value, (uint32_t *)buf);
2683                 if (r)
2684                         return r;
2685
2686                 WREG32(*pos >> 2, value);
2687
2688                 result += 4;
2689                 buf += 4;
2690                 *pos += 4;
2691                 size -= 4;
2692         }
2693
2694         if (use_bank) {
2695                 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
2696                 mutex_unlock(&adev->grbm_idx_mutex);
2697         }
2698
2699         if (pm_pg_lock)
2700                 mutex_unlock(&adev->pm.mutex);
2701
2702         return result;
2703 }
2704
2705 static ssize_t amdgpu_debugfs_regs_pcie_read(struct file *f, char __user *buf,
2706                                         size_t size, loff_t *pos)
2707 {
2708         struct amdgpu_device *adev = file_inode(f)->i_private;
2709         ssize_t result = 0;
2710         int r;
2711
2712         if (size & 0x3 || *pos & 0x3)
2713                 return -EINVAL;
2714
2715         while (size) {
2716                 uint32_t value;
2717
2718                 value = RREG32_PCIE(*pos >> 2);
2719                 r = put_user(value, (uint32_t *)buf);
2720                 if (r)
2721                         return r;
2722
2723                 result += 4;
2724                 buf += 4;
2725                 *pos += 4;
2726                 size -= 4;
2727         }
2728
2729         return result;
2730 }
2731
2732 static ssize_t amdgpu_debugfs_regs_pcie_write(struct file *f, const char __user *buf,
2733                                          size_t size, loff_t *pos)
2734 {
2735         struct amdgpu_device *adev = file_inode(f)->i_private;
2736         ssize_t result = 0;
2737         int r;
2738
2739         if (size & 0x3 || *pos & 0x3)
2740                 return -EINVAL;
2741
2742         while (size) {
2743                 uint32_t value;
2744
2745                 r = get_user(value, (uint32_t *)buf);
2746                 if (r)
2747                         return r;
2748
2749                 WREG32_PCIE(*pos >> 2, value);
2750
2751                 result += 4;
2752                 buf += 4;
2753                 *pos += 4;
2754                 size -= 4;
2755         }
2756
2757         return result;
2758 }
2759
2760 static ssize_t amdgpu_debugfs_regs_didt_read(struct file *f, char __user *buf,
2761                                         size_t size, loff_t *pos)
2762 {
2763         struct amdgpu_device *adev = file_inode(f)->i_private;
2764         ssize_t result = 0;
2765         int r;
2766
2767         if (size & 0x3 || *pos & 0x3)
2768                 return -EINVAL;
2769
2770         while (size) {
2771                 uint32_t value;
2772
2773                 value = RREG32_DIDT(*pos >> 2);
2774                 r = put_user(value, (uint32_t *)buf);
2775                 if (r)
2776                         return r;
2777
2778                 result += 4;
2779                 buf += 4;
2780                 *pos += 4;
2781                 size -= 4;
2782         }
2783
2784         return result;
2785 }
2786
2787 static ssize_t amdgpu_debugfs_regs_didt_write(struct file *f, const char __user *buf,
2788                                          size_t size, loff_t *pos)
2789 {
2790         struct amdgpu_device *adev = file_inode(f)->i_private;
2791         ssize_t result = 0;
2792         int r;
2793
2794         if (size & 0x3 || *pos & 0x3)
2795                 return -EINVAL;
2796
2797         while (size) {
2798                 uint32_t value;
2799
2800                 r = get_user(value, (uint32_t *)buf);
2801                 if (r)
2802                         return r;
2803
2804                 WREG32_DIDT(*pos >> 2, value);
2805
2806                 result += 4;
2807                 buf += 4;
2808                 *pos += 4;
2809                 size -= 4;
2810         }
2811
2812         return result;
2813 }
2814
2815 static ssize_t amdgpu_debugfs_regs_smc_read(struct file *f, char __user *buf,
2816                                         size_t size, loff_t *pos)
2817 {
2818         struct amdgpu_device *adev = file_inode(f)->i_private;
2819         ssize_t result = 0;
2820         int r;
2821
2822         if (size & 0x3 || *pos & 0x3)
2823                 return -EINVAL;
2824
2825         while (size) {
2826                 uint32_t value;
2827
2828                 value = RREG32_SMC(*pos);
2829                 r = put_user(value, (uint32_t *)buf);
2830                 if (r)
2831                         return r;
2832
2833                 result += 4;
2834                 buf += 4;
2835                 *pos += 4;
2836                 size -= 4;
2837         }
2838
2839         return result;
2840 }
2841
2842 static ssize_t amdgpu_debugfs_regs_smc_write(struct file *f, const char __user *buf,
2843                                          size_t size, loff_t *pos)
2844 {
2845         struct amdgpu_device *adev = file_inode(f)->i_private;
2846         ssize_t result = 0;
2847         int r;
2848
2849         if (size & 0x3 || *pos & 0x3)
2850                 return -EINVAL;
2851
2852         while (size) {
2853                 uint32_t value;
2854
2855                 r = get_user(value, (uint32_t *)buf);
2856                 if (r)
2857                         return r;
2858
2859                 WREG32_SMC(*pos, value);
2860
2861                 result += 4;
2862                 buf += 4;
2863                 *pos += 4;
2864                 size -= 4;
2865         }
2866
2867         return result;
2868 }
2869
2870 static ssize_t amdgpu_debugfs_gca_config_read(struct file *f, char __user *buf,
2871                                         size_t size, loff_t *pos)
2872 {
2873         struct amdgpu_device *adev = file_inode(f)->i_private;
2874         ssize_t result = 0;
2875         int r;
2876         uint32_t *config, no_regs = 0;
2877
2878         if (size & 0x3 || *pos & 0x3)
2879                 return -EINVAL;
2880
2881         config = kmalloc_array(256, sizeof(*config), GFP_KERNEL);
2882         if (!config)
2883                 return -ENOMEM;
2884
2885         /* version, increment each time something is added */
2886         config[no_regs++] = 3;
2887         config[no_regs++] = adev->gfx.config.max_shader_engines;
2888         config[no_regs++] = adev->gfx.config.max_tile_pipes;
2889         config[no_regs++] = adev->gfx.config.max_cu_per_sh;
2890         config[no_regs++] = adev->gfx.config.max_sh_per_se;
2891         config[no_regs++] = adev->gfx.config.max_backends_per_se;
2892         config[no_regs++] = adev->gfx.config.max_texture_channel_caches;
2893         config[no_regs++] = adev->gfx.config.max_gprs;
2894         config[no_regs++] = adev->gfx.config.max_gs_threads;
2895         config[no_regs++] = adev->gfx.config.max_hw_contexts;
2896         config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_frontend;
2897         config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_backend;
2898         config[no_regs++] = adev->gfx.config.sc_hiz_tile_fifo_size;
2899         config[no_regs++] = adev->gfx.config.sc_earlyz_tile_fifo_size;
2900         config[no_regs++] = adev->gfx.config.num_tile_pipes;
2901         config[no_regs++] = adev->gfx.config.backend_enable_mask;
2902         config[no_regs++] = adev->gfx.config.mem_max_burst_length_bytes;
2903         config[no_regs++] = adev->gfx.config.mem_row_size_in_kb;
2904         config[no_regs++] = adev->gfx.config.shader_engine_tile_size;
2905         config[no_regs++] = adev->gfx.config.num_gpus;
2906         config[no_regs++] = adev->gfx.config.multi_gpu_tile_size;
2907         config[no_regs++] = adev->gfx.config.mc_arb_ramcfg;
2908         config[no_regs++] = adev->gfx.config.gb_addr_config;
2909         config[no_regs++] = adev->gfx.config.num_rbs;
2910
2911         /* rev==1 */
2912         config[no_regs++] = adev->rev_id;
2913         config[no_regs++] = adev->pg_flags;
2914         config[no_regs++] = adev->cg_flags;
2915
2916         /* rev==2 */
2917         config[no_regs++] = adev->family;
2918         config[no_regs++] = adev->external_rev_id;
2919
2920         /* rev==3 */
2921         config[no_regs++] = adev->pdev->device;
2922         config[no_regs++] = adev->pdev->revision;
2923         config[no_regs++] = adev->pdev->subsystem_device;
2924         config[no_regs++] = adev->pdev->subsystem_vendor;
2925
2926         while (size && (*pos < no_regs * 4)) {
2927                 uint32_t value;
2928
2929                 value = config[*pos >> 2];
2930                 r = put_user(value, (uint32_t *)buf);
2931                 if (r) {
2932                         kfree(config);
2933                         return r;
2934                 }
2935
2936                 result += 4;
2937                 buf += 4;
2938                 *pos += 4;
2939                 size -= 4;
2940         }
2941
2942         kfree(config);
2943         return result;
2944 }
2945
2946 static ssize_t amdgpu_debugfs_sensor_read(struct file *f, char __user *buf,
2947                                         size_t size, loff_t *pos)
2948 {
2949         struct amdgpu_device *adev = file_inode(f)->i_private;
2950         int idx, r;
2951         int32_t value;
2952
2953         if (size != 4 || *pos & 0x3)
2954                 return -EINVAL;
2955
2956         /* convert offset to sensor number */
2957         idx = *pos >> 2;
2958
2959         if (adev->powerplay.pp_funcs && adev->powerplay.pp_funcs->read_sensor)
2960                 r = adev->powerplay.pp_funcs->read_sensor(adev->powerplay.pp_handle, idx, &value);
2961         else
2962                 return -EINVAL;
2963
2964         if (!r)
2965                 r = put_user(value, (int32_t *)buf);
2966
2967         return !r ? 4 : r;
2968 }
2969
2970 static ssize_t amdgpu_debugfs_wave_read(struct file *f, char __user *buf,
2971                                         size_t size, loff_t *pos)
2972 {
2973         struct amdgpu_device *adev = f->f_inode->i_private;
2974         int r, x;
2975         ssize_t result=0;
2976         uint32_t offset, se, sh, cu, wave, simd, data[32];
2977
2978         if (size & 3 || *pos & 3)
2979                 return -EINVAL;
2980
2981         /* decode offset */
2982         offset = (*pos & 0x7F);
2983         se = ((*pos >> 7) & 0xFF);
2984         sh = ((*pos >> 15) & 0xFF);
2985         cu = ((*pos >> 23) & 0xFF);
2986         wave = ((*pos >> 31) & 0xFF);
2987         simd = ((*pos >> 37) & 0xFF);
2988
2989         /* switch to the specific se/sh/cu */
2990         mutex_lock(&adev->grbm_idx_mutex);
2991         amdgpu_gfx_select_se_sh(adev, se, sh, cu);
2992
2993         x = 0;
2994         if (adev->gfx.funcs->read_wave_data)
2995                 adev->gfx.funcs->read_wave_data(adev, simd, wave, data, &x);
2996
2997         amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF);
2998         mutex_unlock(&adev->grbm_idx_mutex);
2999
3000         if (!x)
3001                 return -EINVAL;
3002
3003         while (size && (offset < x * 4)) {
3004                 uint32_t value;
3005
3006                 value = data[offset >> 2];
3007                 r = put_user(value, (uint32_t *)buf);
3008                 if (r)
3009                         return r;
3010
3011                 result += 4;
3012                 buf += 4;
3013                 offset += 4;
3014                 size -= 4;
3015         }
3016
3017         return result;
3018 }
3019
3020 static ssize_t amdgpu_debugfs_gpr_read(struct file *f, char __user *buf,
3021                                         size_t size, loff_t *pos)
3022 {
3023         struct amdgpu_device *adev = f->f_inode->i_private;
3024         int r;
3025         ssize_t result = 0;
3026         uint32_t offset, se, sh, cu, wave, simd, thread, bank, *data;
3027
3028         if (size & 3 || *pos & 3)
3029                 return -EINVAL;
3030
3031         /* decode offset */
3032         offset = (*pos & 0xFFF);       /* in dwords */
3033         se = ((*pos >> 12) & 0xFF);
3034         sh = ((*pos >> 20) & 0xFF);
3035         cu = ((*pos >> 28) & 0xFF);
3036         wave = ((*pos >> 36) & 0xFF);
3037         simd = ((*pos >> 44) & 0xFF);
3038         thread = ((*pos >> 52) & 0xFF);
3039         bank = ((*pos >> 60) & 1);
3040
3041         data = kmalloc_array(1024, sizeof(*data), GFP_KERNEL);
3042         if (!data)
3043                 return -ENOMEM;
3044
3045         /* switch to the specific se/sh/cu */
3046         mutex_lock(&adev->grbm_idx_mutex);
3047         amdgpu_gfx_select_se_sh(adev, se, sh, cu);
3048
3049         if (bank == 0) {
3050                 if (adev->gfx.funcs->read_wave_vgprs)
3051                         adev->gfx.funcs->read_wave_vgprs(adev, simd, wave, thread, offset, size>>2, data);
3052         } else {
3053                 if (adev->gfx.funcs->read_wave_sgprs)
3054                         adev->gfx.funcs->read_wave_sgprs(adev, simd, wave, offset, size>>2, data);
3055         }
3056
3057         amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF);
3058         mutex_unlock(&adev->grbm_idx_mutex);
3059
3060         while (size) {
3061                 uint32_t value;
3062
3063                 value = data[offset++];
3064                 r = put_user(value, (uint32_t *)buf);
3065                 if (r) {
3066                         result = r;
3067                         goto err;
3068                 }
3069
3070                 result += 4;
3071                 buf += 4;
3072                 size -= 4;
3073         }
3074
3075 err:
3076         kfree(data);
3077         return result;
3078 }
3079
3080 static const struct file_operations amdgpu_debugfs_regs_fops = {
3081         .owner = THIS_MODULE,
3082         .read = amdgpu_debugfs_regs_read,
3083         .write = amdgpu_debugfs_regs_write,
3084         .llseek = default_llseek
3085 };
3086 static const struct file_operations amdgpu_debugfs_regs_didt_fops = {
3087         .owner = THIS_MODULE,
3088         .read = amdgpu_debugfs_regs_didt_read,
3089         .write = amdgpu_debugfs_regs_didt_write,
3090         .llseek = default_llseek
3091 };
3092 static const struct file_operations amdgpu_debugfs_regs_pcie_fops = {
3093         .owner = THIS_MODULE,
3094         .read = amdgpu_debugfs_regs_pcie_read,
3095         .write = amdgpu_debugfs_regs_pcie_write,
3096         .llseek = default_llseek
3097 };
3098 static const struct file_operations amdgpu_debugfs_regs_smc_fops = {
3099         .owner = THIS_MODULE,
3100         .read = amdgpu_debugfs_regs_smc_read,
3101         .write = amdgpu_debugfs_regs_smc_write,
3102         .llseek = default_llseek
3103 };
3104
3105 static const struct file_operations amdgpu_debugfs_gca_config_fops = {
3106         .owner = THIS_MODULE,
3107         .read = amdgpu_debugfs_gca_config_read,
3108         .llseek = default_llseek
3109 };
3110
3111 static const struct file_operations amdgpu_debugfs_sensors_fops = {
3112         .owner = THIS_MODULE,
3113         .read = amdgpu_debugfs_sensor_read,
3114         .llseek = default_llseek
3115 };
3116
3117 static const struct file_operations amdgpu_debugfs_wave_fops = {
3118         .owner = THIS_MODULE,
3119         .read = amdgpu_debugfs_wave_read,
3120         .llseek = default_llseek
3121 };
3122 static const struct file_operations amdgpu_debugfs_gpr_fops = {
3123         .owner = THIS_MODULE,
3124         .read = amdgpu_debugfs_gpr_read,
3125         .llseek = default_llseek
3126 };
3127
3128 static const struct file_operations *debugfs_regs[] = {
3129         &amdgpu_debugfs_regs_fops,
3130         &amdgpu_debugfs_regs_didt_fops,
3131         &amdgpu_debugfs_regs_pcie_fops,
3132         &amdgpu_debugfs_regs_smc_fops,
3133         &amdgpu_debugfs_gca_config_fops,
3134         &amdgpu_debugfs_sensors_fops,
3135         &amdgpu_debugfs_wave_fops,
3136         &amdgpu_debugfs_gpr_fops,
3137 };
3138
3139 static const char *debugfs_regs_names[] = {
3140         "amdgpu_regs",
3141         "amdgpu_regs_didt",
3142         "amdgpu_regs_pcie",
3143         "amdgpu_regs_smc",
3144         "amdgpu_gca_config",
3145         "amdgpu_sensors",
3146         "amdgpu_wave",
3147         "amdgpu_gpr",
3148 };
3149
3150 static int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
3151 {
3152         struct drm_minor *minor = adev->ddev->primary;
3153         struct dentry *ent, *root = minor->debugfs_root;
3154         unsigned i, j;
3155
3156         for (i = 0; i < ARRAY_SIZE(debugfs_regs); i++) {
3157                 ent = debugfs_create_file(debugfs_regs_names[i],
3158                                           S_IFREG | S_IRUGO, root,
3159                                           adev, debugfs_regs[i]);
3160                 if (IS_ERR(ent)) {
3161                         for (j = 0; j < i; j++) {
3162                                 debugfs_remove(adev->debugfs_regs[i]);
3163                                 adev->debugfs_regs[i] = NULL;
3164                         }
3165                         return PTR_ERR(ent);
3166                 }
3167
3168                 if (!i)
3169                         i_size_write(ent->d_inode, adev->rmmio_size);
3170                 adev->debugfs_regs[i] = ent;
3171         }
3172
3173         return 0;
3174 }
3175
3176 static void amdgpu_debugfs_regs_cleanup(struct amdgpu_device *adev)
3177 {
3178         unsigned i;
3179
3180         for (i = 0; i < ARRAY_SIZE(debugfs_regs); i++) {
3181                 if (adev->debugfs_regs[i]) {
3182                         debugfs_remove(adev->debugfs_regs[i]);
3183                         adev->debugfs_regs[i] = NULL;
3184                 }
3185         }
3186 }
3187
3188 int amdgpu_debugfs_init(struct drm_minor *minor)
3189 {
3190         return 0;
3191 }
3192 #else
3193 static int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
3194 {
3195         return 0;
3196 }
3197 static void amdgpu_debugfs_regs_cleanup(struct amdgpu_device *adev) { }
3198 #endif