drivers/gpu/drm/amd/amdgpu/amdgpu_device.c

   1 /*
   2  * Copyright 2008 Advanced Micro Devices, Inc.
   3  * Copyright 2008 Red Hat Inc.
   4  * Copyright 2009 Jerome Glisse.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22  * OTHER DEALINGS IN THE SOFTWARE.
  23  *
  24  * Authors: Dave Airlie
  25  *          Alex Deucher
  26  *          Jerome Glisse
  27  */
  28 #include <linux/power_supply.h>
  29 #include <linux/kthread.h>
  30 #include <linux/module.h>
  31 #include <linux/console.h>
  32 #include <linux/slab.h>
  33 #include <linux/iommu.h>
  34 #include <linux/pci.h>
  35 #include <linux/devcoredump.h>
  36 #include <generated/utsrelease.h>
  37 #include <linux/pci-p2pdma.h>
  38 #include <linux/apple-gmux.h>
  39
  40 #include <drm/drm_aperture.h>
  41 #include <drm/drm_atomic_helper.h>
  42 #include <drm/drm_crtc_helper.h>
  43 #include <drm/drm_fb_helper.h>
  44 #include <drm/drm_probe_helper.h>
  45 #include <drm/amdgpu_drm.h>
  46 #include <linux/vgaarb.h>
  47 #include <linux/vga_switcheroo.h>
  48 #include <linux/efi.h>
  49 #include "amdgpu.h"
  50 #include "amdgpu_trace.h"
  51 #include "amdgpu_i2c.h"
  52 #include "atom.h"
  53 #include "amdgpu_atombios.h"
  54 #include "amdgpu_atomfirmware.h"
  55 #include "amd_pcie.h"
  56 #ifdef CONFIG_DRM_AMDGPU_SI
  57 #include "si.h"
  58 #endif
  59 #ifdef CONFIG_DRM_AMDGPU_CIK
  60 #include "cik.h"
  61 #endif
  62 #include "vi.h"
  63 #include "soc15.h"
  64 #include "nv.h"
  65 #include "bif/bif_4_1_d.h"
  66 #include <linux/firmware.h>
  67 #include "amdgpu_vf_error.h"
  68
  69 #include "amdgpu_amdkfd.h"
  70 #include "amdgpu_pm.h"
  71
  72 #include "amdgpu_xgmi.h"
  73 #include "amdgpu_ras.h"
  74 #include "amdgpu_pmu.h"
  75 #include "amdgpu_fru_eeprom.h"
  76 #include "amdgpu_reset.h"
  77
  78 #include <linux/suspend.h>
  79 #include <drm/task_barrier.h>
  80 #include <linux/pm_runtime.h>
  81
  82 #include <drm/drm_drv.h>
  83
  84 #if IS_ENABLED(CONFIG_X86)
  85 #include <asm/intel-family.h>
  86 #endif
  87
  88 MODULE_FIRMWARE("amdgpu/vega10_gpu_info.bin");
  89 MODULE_FIRMWARE("amdgpu/vega12_gpu_info.bin");
  90 MODULE_FIRMWARE("amdgpu/raven_gpu_info.bin");
  91 MODULE_FIRMWARE("amdgpu/picasso_gpu_info.bin");
  92 MODULE_FIRMWARE("amdgpu/raven2_gpu_info.bin");
  93 MODULE_FIRMWARE("amdgpu/arcturus_gpu_info.bin");
  94 MODULE_FIRMWARE("amdgpu/navi12_gpu_info.bin");
  95
  96 #define AMDGPU_RESUME_MS                2000
  97 #define AMDGPU_MAX_RETRY_LIMIT          2
  98 #define AMDGPU_RETRY_SRIOV_RESET(r) ((r) == -EBUSY || (r) == -ETIMEDOUT || (r) == -EINVAL)
  99
 100 static const struct drm_driver amdgpu_kms_driver;
 101
 102 const char *amdgpu_asic_name[] = {
 103         "TAHITI",
 104         "PITCAIRN",
 105         "VERDE",
 106         "OLAND",
 107         "HAINAN",
 108         "BONAIRE",
 109         "KAVERI",
 110         "KABINI",
 111         "HAWAII",
 112         "MULLINS",
 113         "TOPAZ",
 114         "TONGA",
 115         "FIJI",
 116         "CARRIZO",
 117         "STONEY",
 118         "POLARIS10",
 119         "POLARIS11",
 120         "POLARIS12",
 121         "VEGAM",
 122         "VEGA10",
 123         "VEGA12",
 124         "VEGA20",
 125         "RAVEN",
 126         "ARCTURUS",
 127         "RENOIR",
 128         "ALDEBARAN",
 129         "NAVI10",
 130         "CYAN_SKILLFISH",
 131         "NAVI14",
 132         "NAVI12",
 133         "SIENNA_CICHLID",
 134         "NAVY_FLOUNDER",
 135         "VANGOGH",
 136         "DIMGREY_CAVEFISH",
 137         "BEIGE_GOBY",
 138         "YELLOW_CARP",
 139         "IP DISCOVERY",
 140         "LAST",
 141 };
 142
 143 /**
 144  * DOC: pcie_replay_count
 145  *
 146  * The amdgpu driver provides a sysfs API for reporting the total number
 147  * of PCIe replays (NAKs)
 148  * The file pcie_replay_count is used for this and returns the total
 149  * number of replays as a sum of the NAKs generated and NAKs received
 150  */
 151
 152 static ssize_t amdgpu_device_get_pcie_replay_count(struct device *dev,
 153                 struct device_attribute *attr, char *buf)
 154 {
 155         struct drm_device *ddev = dev_get_drvdata(dev);
 156         struct amdgpu_device *adev = drm_to_adev(ddev);
 157         uint64_t cnt = amdgpu_asic_get_pcie_replay_count(adev);
 158
 159         return sysfs_emit(buf, "%llu\n", cnt);
 160 }
 161
 162 static DEVICE_ATTR(pcie_replay_count, S_IRUGO,
 163                 amdgpu_device_get_pcie_replay_count, NULL);
 164
 165 static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev);
 166
 167 /**
 168  * DOC: product_name
 169  *
 170  * The amdgpu driver provides a sysfs API for reporting the product name
 171  * for the device
 172  * The file product_name is used for this and returns the product name
 173  * as returned from the FRU.
 174  * NOTE: This is only available for certain server cards
 175  */
 176
 177 static ssize_t amdgpu_device_get_product_name(struct device *dev,
 178                 struct device_attribute *attr, char *buf)
 179 {
 180         struct drm_device *ddev = dev_get_drvdata(dev);
 181         struct amdgpu_device *adev = drm_to_adev(ddev);
 182
 183         return sysfs_emit(buf, "%s\n", adev->product_name);
 184 }
 185
 186 static DEVICE_ATTR(product_name, S_IRUGO,
 187                 amdgpu_device_get_product_name, NULL);
 188
 189 /**
 190  * DOC: product_number
 191  *
 192  * The amdgpu driver provides a sysfs API for reporting the part number
 193  * for the device
 194  * The file product_number is used for this and returns the part number
 195  * as returned from the FRU.
 196  * NOTE: This is only available for certain server cards
 197  */
 198
 199 static ssize_t amdgpu_device_get_product_number(struct device *dev,
 200                 struct device_attribute *attr, char *buf)
 201 {
 202         struct drm_device *ddev = dev_get_drvdata(dev);
 203         struct amdgpu_device *adev = drm_to_adev(ddev);
 204
 205         return sysfs_emit(buf, "%s\n", adev->product_number);
 206 }
 207
 208 static DEVICE_ATTR(product_number, S_IRUGO,
 209                 amdgpu_device_get_product_number, NULL);
 210
 211 /**
 212  * DOC: serial_number
 213  *
 214  * The amdgpu driver provides a sysfs API for reporting the serial number
 215  * for the device
 216  * The file serial_number is used for this and returns the serial number
 217  * as returned from the FRU.
 218  * NOTE: This is only available for certain server cards
 219  */
 220
 221 static ssize_t amdgpu_device_get_serial_number(struct device *dev,
 222                 struct device_attribute *attr, char *buf)
 223 {
 224         struct drm_device *ddev = dev_get_drvdata(dev);
 225         struct amdgpu_device *adev = drm_to_adev(ddev);
 226
 227         return sysfs_emit(buf, "%s\n", adev->serial);
 228 }
 229
 230 static DEVICE_ATTR(serial_number, S_IRUGO,
 231                 amdgpu_device_get_serial_number, NULL);
 232
 233 /**
 234  * amdgpu_device_supports_px - Is the device a dGPU with ATPX power control
 235  *
 236  * @dev: drm_device pointer
 237  *
 238  * Returns true if the device is a dGPU with ATPX power control,
 239  * otherwise return false.
 240  */
 241 bool amdgpu_device_supports_px(struct drm_device *dev)
 242 {
 243         struct amdgpu_device *adev = drm_to_adev(dev);
 244
 245         if ((adev->flags & AMD_IS_PX) && !amdgpu_is_atpx_hybrid())
 246                 return true;
 247         return false;
 248 }
 249
 250 /**
 251  * amdgpu_device_supports_boco - Is the device a dGPU with ACPI power resources
 252  *
 253  * @dev: drm_device pointer
 254  *
 255  * Returns true if the device is a dGPU with ACPI power control,
 256  * otherwise return false.
 257  */
 258 bool amdgpu_device_supports_boco(struct drm_device *dev)
 259 {
 260         struct amdgpu_device *adev = drm_to_adev(dev);
 261
 262         if (adev->has_pr3 ||
 263             ((adev->flags & AMD_IS_PX) && amdgpu_is_atpx_hybrid()))
 264                 return true;
 265         return false;
 266 }
 267
 268 /**
 269  * amdgpu_device_supports_baco - Does the device support BACO
 270  *
 271  * @dev: drm_device pointer
 272  *
 273  * Returns true if the device supporte BACO,
 274  * otherwise return false.
 275  */
 276 bool amdgpu_device_supports_baco(struct drm_device *dev)
 277 {
 278         struct amdgpu_device *adev = drm_to_adev(dev);
 279
 280         return amdgpu_asic_supports_baco(adev);
 281 }
 282
 283 /**
 284  * amdgpu_device_supports_smart_shift - Is the device dGPU with
 285  * smart shift support
 286  *
 287  * @dev: drm_device pointer
 288  *
 289  * Returns true if the device is a dGPU with Smart Shift support,
 290  * otherwise returns false.
 291  */
 292 bool amdgpu_device_supports_smart_shift(struct drm_device *dev)
 293 {
 294         return (amdgpu_device_supports_boco(dev) &&
 295                 amdgpu_acpi_is_power_shift_control_supported());
 296 }
 297
 298 /*
 299  * VRAM access helper functions
 300  */
 301
 302 /**
 303  * amdgpu_device_mm_access - access vram by MM_INDEX/MM_DATA
 304  *
 305  * @adev: amdgpu_device pointer
 306  * @pos: offset of the buffer in vram
 307  * @buf: virtual address of the buffer in system memory
 308  * @size: read/write size, sizeof(@buf) must > @size
 309  * @write: true - write to vram, otherwise - read from vram
 310  */
 311 void amdgpu_device_mm_access(struct amdgpu_device *adev, loff_t pos,
 312                              void *buf, size_t size, bool write)
 313 {
 314         unsigned long flags;
 315         uint32_t hi = ~0, tmp = 0;
 316         uint32_t *data = buf;
 317         uint64_t last;
 318         int idx;
 319
 320         if (!drm_dev_enter(adev_to_drm(adev), &idx))
 321                 return;
 322
 323         BUG_ON(!IS_ALIGNED(pos, 4) || !IS_ALIGNED(size, 4));
 324
 325         spin_lock_irqsave(&adev->mmio_idx_lock, flags);
 326         for (last = pos + size; pos < last; pos += 4) {
 327                 tmp = pos >> 31;
 328
 329                 WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)pos) | 0x80000000);
 330                 if (tmp != hi) {
 331                         WREG32_NO_KIQ(mmMM_INDEX_HI, tmp);
 332                         hi = tmp;
 333                 }
 334                 if (write)
 335                         WREG32_NO_KIQ(mmMM_DATA, *data++);
 336                 else
 337                         *data++ = RREG32_NO_KIQ(mmMM_DATA);
 338         }
 339
 340         spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
 341         drm_dev_exit(idx);
 342 }
 343
 344 /**
 345  * amdgpu_device_aper_access - access vram by vram aperature
 346  *
 347  * @adev: amdgpu_device pointer
 348  * @pos: offset of the buffer in vram
 349  * @buf: virtual address of the buffer in system memory
 350  * @size: read/write size, sizeof(@buf) must > @size
 351  * @write: true - write to vram, otherwise - read from vram
 352  *
 353  * The return value means how many bytes have been transferred.
 354  */
 355 size_t amdgpu_device_aper_access(struct amdgpu_device *adev, loff_t pos,
 356                                  void *buf, size_t size, bool write)
 357 {
 358 #ifdef CONFIG_64BIT
 359         void __iomem *addr;
 360         size_t count = 0;
 361         uint64_t last;
 362
 363         if (!adev->mman.aper_base_kaddr)
 364                 return 0;
 365
 366         last = min(pos + size, adev->gmc.visible_vram_size);
 367         if (last > pos) {
 368                 addr = adev->mman.aper_base_kaddr + pos;
 369                 count = last - pos;
 370
 371                 if (write) {
 372                         memcpy_toio(addr, buf, count);
 373                         mb();
 374                         amdgpu_device_flush_hdp(adev, NULL);
 375                 } else {
 376                         amdgpu_device_invalidate_hdp(adev, NULL);
 377                         mb();
 378                         memcpy_fromio(buf, addr, count);
 379                 }
 380
 381         }
 382
 383         return count;
 384 #else
 385         return 0;
 386 #endif
 387 }
 388
 389 /**
 390  * amdgpu_device_vram_access - read/write a buffer in vram
 391  *
 392  * @adev: amdgpu_device pointer
 393  * @pos: offset of the buffer in vram
 394  * @buf: virtual address of the buffer in system memory
 395  * @size: read/write size, sizeof(@buf) must > @size
 396  * @write: true - write to vram, otherwise - read from vram
 397  */
 398 void amdgpu_device_vram_access(struct amdgpu_device *adev, loff_t pos,
 399                                void *buf, size_t size, bool write)
 400 {
 401         size_t count;
 402
 403         /* try to using vram apreature to access vram first */
 404         count = amdgpu_device_aper_access(adev, pos, buf, size, write);
 405         size -= count;
 406         if (size) {
 407                 /* using MM to access rest vram */
 408                 pos += count;
 409                 buf += count;
 410                 amdgpu_device_mm_access(adev, pos, buf, size, write);
 411         }
 412 }
 413
 414 /*
 415  * register access helper functions.
 416  */
 417
 418 /* Check if hw access should be skipped because of hotplug or device error */
 419 bool amdgpu_device_skip_hw_access(struct amdgpu_device *adev)
 420 {
 421         if (adev->no_hw_access)
 422                 return true;
 423
 424 #ifdef CONFIG_LOCKDEP
 425         /*
 426          * This is a bit complicated to understand, so worth a comment. What we assert
 427          * here is that the GPU reset is not running on another thread in parallel.
 428          *
 429          * For this we trylock the read side of the reset semaphore, if that succeeds
 430          * we know that the reset is not running in paralell.
 431          *
 432          * If the trylock fails we assert that we are either already holding the read
 433          * side of the lock or are the reset thread itself and hold the write side of
 434          * the lock.
 435          */
 436         if (in_task()) {
 437                 if (down_read_trylock(&adev->reset_domain->sem))
 438                         up_read(&adev->reset_domain->sem);
 439                 else
 440                         lockdep_assert_held(&adev->reset_domain->sem);
 441         }
 442 #endif
 443         return false;
 444 }
 445
 446 /**
 447  * amdgpu_device_rreg - read a memory mapped IO or indirect register
 448  *
 449  * @adev: amdgpu_device pointer
 450  * @reg: dword aligned register offset
 451  * @acc_flags: access flags which require special behavior
 452  *
 453  * Returns the 32 bit value from the offset specified.
 454  */
 455 uint32_t amdgpu_device_rreg(struct amdgpu_device *adev,
 456                             uint32_t reg, uint32_t acc_flags)
 457 {
 458         uint32_t ret;
 459
 460         if (amdgpu_device_skip_hw_access(adev))
 461                 return 0;
 462
 463         if ((reg * 4) < adev->rmmio_size) {
 464                 if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
 465                     amdgpu_sriov_runtime(adev) &&
 466                     down_read_trylock(&adev->reset_domain->sem)) {
 467                         ret = amdgpu_kiq_rreg(adev, reg);
 468                         up_read(&adev->reset_domain->sem);
 469                 } else {
 470                         ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
 471                 }
 472         } else {
 473                 ret = adev->pcie_rreg(adev, reg * 4);
 474         }
 475
 476         trace_amdgpu_device_rreg(adev->pdev->device, reg, ret);
 477
 478         return ret;
 479 }
 480
 481 /*
 482  * MMIO register read with bytes helper functions
 483  * @offset:bytes offset from MMIO start
 484  *
 485 */
 486
 487 /**
 488  * amdgpu_mm_rreg8 - read a memory mapped IO register
 489  *
 490  * @adev: amdgpu_device pointer
 491  * @offset: byte aligned register offset
 492  *
 493  * Returns the 8 bit value from the offset specified.
 494  */
 495 uint8_t amdgpu_mm_rreg8(struct amdgpu_device *adev, uint32_t offset)
 496 {
 497         if (amdgpu_device_skip_hw_access(adev))
 498                 return 0;
 499
 500         if (offset < adev->rmmio_size)
 501                 return (readb(adev->rmmio + offset));
 502         BUG();
 503 }
 504
 505 /*
 506  * MMIO register write with bytes helper functions
 507  * @offset:bytes offset from MMIO start
 508  * @value: the value want to be written to the register
 509  *
 510 */
 511 /**
 512  * amdgpu_mm_wreg8 - read a memory mapped IO register
 513  *
 514  * @adev: amdgpu_device pointer
 515  * @offset: byte aligned register offset
 516  * @value: 8 bit value to write
 517  *
 518  * Writes the value specified to the offset specified.
 519  */
 520 void amdgpu_mm_wreg8(struct amdgpu_device *adev, uint32_t offset, uint8_t value)
 521 {
 522         if (amdgpu_device_skip_hw_access(adev))
 523                 return;
 524
 525         if (offset < adev->rmmio_size)
 526                 writeb(value, adev->rmmio + offset);
 527         else
 528                 BUG();
 529 }
 530
 531 /**
 532  * amdgpu_device_wreg - write to a memory mapped IO or indirect register
 533  *
 534  * @adev: amdgpu_device pointer
 535  * @reg: dword aligned register offset
 536  * @v: 32 bit value to write to the register
 537  * @acc_flags: access flags which require special behavior
 538  *
 539  * Writes the value specified to the offset specified.
 540  */
 541 void amdgpu_device_wreg(struct amdgpu_device *adev,
 542                         uint32_t reg, uint32_t v,
 543                         uint32_t acc_flags)
 544 {
 545         if (amdgpu_device_skip_hw_access(adev))
 546                 return;
 547
 548         if ((reg * 4) < adev->rmmio_size) {
 549                 if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
 550                     amdgpu_sriov_runtime(adev) &&
 551                     down_read_trylock(&adev->reset_domain->sem)) {
 552                         amdgpu_kiq_wreg(adev, reg, v);
 553                         up_read(&adev->reset_domain->sem);
 554                 } else {
 555                         writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
 556                 }
 557         } else {
 558                 adev->pcie_wreg(adev, reg * 4, v);
 559         }
 560
 561         trace_amdgpu_device_wreg(adev->pdev->device, reg, v);
 562 }
 563
 564 /**
 565  * amdgpu_mm_wreg_mmio_rlc -  write register either with direct/indirect mmio or with RLC path if in range
 566  *
 567  * @adev: amdgpu_device pointer
 568  * @reg: mmio/rlc register
 569  * @v: value to write
 570  *
 571  * this function is invoked only for the debugfs register access
 572  */
 573 void amdgpu_mm_wreg_mmio_rlc(struct amdgpu_device *adev,
 574                              uint32_t reg, uint32_t v)
 575 {
 576         if (amdgpu_device_skip_hw_access(adev))
 577                 return;
 578
 579         if (amdgpu_sriov_fullaccess(adev) &&
 580             adev->gfx.rlc.funcs &&
 581             adev->gfx.rlc.funcs->is_rlcg_access_range) {
 582                 if (adev->gfx.rlc.funcs->is_rlcg_access_range(adev, reg))
 583                         return amdgpu_sriov_wreg(adev, reg, v, 0, 0);
 584         } else if ((reg * 4) >= adev->rmmio_size) {
 585                 adev->pcie_wreg(adev, reg * 4, v);
 586         } else {
 587                 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
 588         }
 589 }
 590
 591 /**
 592  * amdgpu_mm_rdoorbell - read a doorbell dword
 593  *
 594  * @adev: amdgpu_device pointer
 595  * @index: doorbell index
 596  *
 597  * Returns the value in the doorbell aperture at the
 598  * requested doorbell index (CIK).
 599  */
 600 u32 amdgpu_mm_rdoorbell(struct amdgpu_device *adev, u32 index)
 601 {
 602         if (amdgpu_device_skip_hw_access(adev))
 603                 return 0;
 604
 605         if (index < adev->doorbell.num_kernel_doorbells) {
 606                 return readl(adev->doorbell.ptr + index);
 607         } else {
 608                 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
 609                 return 0;
 610         }
 611 }
 612
 613 /**
 614  * amdgpu_mm_wdoorbell - write a doorbell dword
 615  *
 616  * @adev: amdgpu_device pointer
 617  * @index: doorbell index
 618  * @v: value to write
 619  *
 620  * Writes @v to the doorbell aperture at the
 621  * requested doorbell index (CIK).
 622  */
 623 void amdgpu_mm_wdoorbell(struct amdgpu_device *adev, u32 index, u32 v)
 624 {
 625         if (amdgpu_device_skip_hw_access(adev))
 626                 return;
 627
 628         if (index < adev->doorbell.num_kernel_doorbells) {
 629                 writel(v, adev->doorbell.ptr + index);
 630         } else {
 631                 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
 632         }
 633 }
 634
 635 /**
 636  * amdgpu_mm_rdoorbell64 - read a doorbell Qword
 637  *
 638  * @adev: amdgpu_device pointer
 639  * @index: doorbell index
 640  *
 641  * Returns the value in the doorbell aperture at the
 642  * requested doorbell index (VEGA10+).
 643  */
 644 u64 amdgpu_mm_rdoorbell64(struct amdgpu_device *adev, u32 index)
 645 {
 646         if (amdgpu_device_skip_hw_access(adev))
 647                 return 0;
 648
 649         if (index < adev->doorbell.num_kernel_doorbells) {
 650                 return atomic64_read((atomic64_t *)(adev->doorbell.ptr + index));
 651         } else {
 652                 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
 653                 return 0;
 654         }
 655 }
 656
 657 /**
 658  * amdgpu_mm_wdoorbell64 - write a doorbell Qword
 659  *
 660  * @adev: amdgpu_device pointer
 661  * @index: doorbell index
 662  * @v: value to write
 663  *
 664  * Writes @v to the doorbell aperture at the
 665  * requested doorbell index (VEGA10+).
 666  */
 667 void amdgpu_mm_wdoorbell64(struct amdgpu_device *adev, u32 index, u64 v)
 668 {
 669         if (amdgpu_device_skip_hw_access(adev))
 670                 return;
 671
 672         if (index < adev->doorbell.num_kernel_doorbells) {
 673                 atomic64_set((atomic64_t *)(adev->doorbell.ptr + index), v);
 674         } else {
 675                 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
 676         }
 677 }
 678
 679 /**
 680  * amdgpu_device_indirect_rreg - read an indirect register
 681  *
 682  * @adev: amdgpu_device pointer
 683  * @reg_addr: indirect register address to read from
 684  *
 685  * Returns the value of indirect register @reg_addr
 686  */
 687 u32 amdgpu_device_indirect_rreg(struct amdgpu_device *adev,
 688                                 u32 reg_addr)
 689 {
 690         unsigned long flags, pcie_index, pcie_data;
 691         void __iomem *pcie_index_offset;
 692         void __iomem *pcie_data_offset;
 693         u32 r;
 694
 695         pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
 696         pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
 697
 698         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 699         pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
 700         pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
 701
 702         writel(reg_addr, pcie_index_offset);
 703         readl(pcie_index_offset);
 704         r = readl(pcie_data_offset);
 705         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 706
 707         return r;
 708 }
 709
 710 /**
 711  * amdgpu_device_indirect_rreg64 - read a 64bits indirect register
 712  *
 713  * @adev: amdgpu_device pointer
 714  * @reg_addr: indirect register address to read from
 715  *
 716  * Returns the value of indirect register @reg_addr
 717  */
 718 u64 amdgpu_device_indirect_rreg64(struct amdgpu_device *adev,
 719                                   u32 reg_addr)
 720 {
 721         unsigned long flags, pcie_index, pcie_data;
 722         void __iomem *pcie_index_offset;
 723         void __iomem *pcie_data_offset;
 724         u64 r;
 725
 726         pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
 727         pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
 728
 729         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 730         pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
 731         pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
 732
 733         /* read low 32 bits */
 734         writel(reg_addr, pcie_index_offset);
 735         readl(pcie_index_offset);
 736         r = readl(pcie_data_offset);
 737         /* read high 32 bits */
 738         writel(reg_addr + 4, pcie_index_offset);
 739         readl(pcie_index_offset);
 740         r |= ((u64)readl(pcie_data_offset) << 32);
 741         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 742
 743         return r;
 744 }
 745
 746 /**
 747  * amdgpu_device_indirect_wreg - write an indirect register address
 748  *
 749  * @adev: amdgpu_device pointer
 750  * @pcie_index: mmio register offset
 751  * @pcie_data: mmio register offset
 752  * @reg_addr: indirect register offset
 753  * @reg_data: indirect register data
 754  *
 755  */
 756 void amdgpu_device_indirect_wreg(struct amdgpu_device *adev,
 757                                  u32 reg_addr, u32 reg_data)
 758 {
 759         unsigned long flags, pcie_index, pcie_data;
 760         void __iomem *pcie_index_offset;
 761         void __iomem *pcie_data_offset;
 762
 763         pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
 764         pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
 765
 766         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 767         pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
 768         pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
 769
 770         writel(reg_addr, pcie_index_offset);
 771         readl(pcie_index_offset);
 772         writel(reg_data, pcie_data_offset);
 773         readl(pcie_data_offset);
 774         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 775 }
 776
 777 /**
 778  * amdgpu_device_indirect_wreg64 - write a 64bits indirect register address
 779  *
 780  * @adev: amdgpu_device pointer
 781  * @pcie_index: mmio register offset
 782  * @pcie_data: mmio register offset
 783  * @reg_addr: indirect register offset
 784  * @reg_data: indirect register data
 785  *
 786  */
 787 void amdgpu_device_indirect_wreg64(struct amdgpu_device *adev,
 788                                    u32 reg_addr, u64 reg_data)
 789 {
 790         unsigned long flags, pcie_index, pcie_data;
 791         void __iomem *pcie_index_offset;
 792         void __iomem *pcie_data_offset;
 793
 794         pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
 795         pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
 796
 797         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 798         pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
 799         pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
 800
 801         /* write low 32 bits */
 802         writel(reg_addr, pcie_index_offset);
 803         readl(pcie_index_offset);
 804         writel((u32)(reg_data & 0xffffffffULL), pcie_data_offset);
 805         readl(pcie_data_offset);
 806         /* write high 32 bits */
 807         writel(reg_addr + 4, pcie_index_offset);
 808         readl(pcie_index_offset);
 809         writel((u32)(reg_data >> 32), pcie_data_offset);
 810         readl(pcie_data_offset);
 811         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 812 }
 813
 814 /**
 815  * amdgpu_device_get_rev_id - query device rev_id
 816  *
 817  * @adev: amdgpu_device pointer
 818  *
 819  * Return device rev_id
 820  */
 821 u32 amdgpu_device_get_rev_id(struct amdgpu_device *adev)
 822 {
 823         return adev->nbio.funcs->get_rev_id(adev);
 824 }
 825
 826 /**
 827  * amdgpu_invalid_rreg - dummy reg read function
 828  *
 829  * @adev: amdgpu_device pointer
 830  * @reg: offset of register
 831  *
 832  * Dummy register read function.  Used for register blocks
 833  * that certain asics don't have (all asics).
 834  * Returns the value in the register.
 835  */
 836 static uint32_t amdgpu_invalid_rreg(struct amdgpu_device *adev, uint32_t reg)
 837 {
 838         DRM_ERROR("Invalid callback to read register 0x%04X\n", reg);
 839         BUG();
 840         return 0;
 841 }
 842
 843 /**
 844  * amdgpu_invalid_wreg - dummy reg write function
 845  *
 846  * @adev: amdgpu_device pointer
 847  * @reg: offset of register
 848  * @v: value to write to the register
 849  *
 850  * Dummy register read function.  Used for register blocks
 851  * that certain asics don't have (all asics).
 852  */
 853 static void amdgpu_invalid_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
 854 {
 855         DRM_ERROR("Invalid callback to write register 0x%04X with 0x%08X\n",
 856                   reg, v);
 857         BUG();
 858 }
 859
 860 /**
 861  * amdgpu_invalid_rreg64 - dummy 64 bit reg read function
 862  *
 863  * @adev: amdgpu_device pointer
 864  * @reg: offset of register
 865  *
 866  * Dummy register read function.  Used for register blocks
 867  * that certain asics don't have (all asics).
 868  * Returns the value in the register.
 869  */
 870 static uint64_t amdgpu_invalid_rreg64(struct amdgpu_device *adev, uint32_t reg)
 871 {
 872         DRM_ERROR("Invalid callback to read 64 bit register 0x%04X\n", reg);
 873         BUG();
 874         return 0;
 875 }
 876
 877 /**
 878  * amdgpu_invalid_wreg64 - dummy reg write function
 879  *
 880  * @adev: amdgpu_device pointer
 881  * @reg: offset of register
 882  * @v: value to write to the register
 883  *
 884  * Dummy register read function.  Used for register blocks
 885  * that certain asics don't have (all asics).
 886  */
 887 static void amdgpu_invalid_wreg64(struct amdgpu_device *adev, uint32_t reg, uint64_t v)
 888 {
 889         DRM_ERROR("Invalid callback to write 64 bit register 0x%04X with 0x%08llX\n",
 890                   reg, v);
 891         BUG();
 892 }
 893
 894 /**
 895  * amdgpu_block_invalid_rreg - dummy reg read function
 896  *
 897  * @adev: amdgpu_device pointer
 898  * @block: offset of instance
 899  * @reg: offset of register
 900  *
 901  * Dummy register read function.  Used for register blocks
 902  * that certain asics don't have (all asics).
 903  * Returns the value in the register.
 904  */
 905 static uint32_t amdgpu_block_invalid_rreg(struct amdgpu_device *adev,
 906                                           uint32_t block, uint32_t reg)
 907 {
 908         DRM_ERROR("Invalid callback to read register 0x%04X in block 0x%04X\n",
 909                   reg, block);
 910         BUG();
 911         return 0;
 912 }
 913
 914 /**
 915  * amdgpu_block_invalid_wreg - dummy reg write function
 916  *
 917  * @adev: amdgpu_device pointer
 918  * @block: offset of instance
 919  * @reg: offset of register
 920  * @v: value to write to the register
 921  *
 922  * Dummy register read function.  Used for register blocks
 923  * that certain asics don't have (all asics).
 924  */
 925 static void amdgpu_block_invalid_wreg(struct amdgpu_device *adev,
 926                                       uint32_t block,
 927                                       uint32_t reg, uint32_t v)
 928 {
 929         DRM_ERROR("Invalid block callback to write register 0x%04X in block 0x%04X with 0x%08X\n",
 930                   reg, block, v);
 931         BUG();
 932 }
 933
 934 /**
 935  * amdgpu_device_asic_init - Wrapper for atom asic_init
 936  *
 937  * @adev: amdgpu_device pointer
 938  *
 939  * Does any asic specific work and then calls atom asic init.
 940  */
 941 static int amdgpu_device_asic_init(struct amdgpu_device *adev)
 942 {
 943         amdgpu_asic_pre_asic_init(adev);
 944
 945         if (adev->ip_versions[GC_HWIP][0] >= IP_VERSION(11, 0, 0))
 946                 return amdgpu_atomfirmware_asic_init(adev, true);
 947         else
 948                 return amdgpu_atom_asic_init(adev->mode_info.atom_context);
 949 }
 950
 951 /**
 952  * amdgpu_device_mem_scratch_init - allocate the VRAM scratch page
 953  *
 954  * @adev: amdgpu_device pointer
 955  *
 956  * Allocates a scratch page of VRAM for use by various things in the
 957  * driver.
 958  */
 959 static int amdgpu_device_mem_scratch_init(struct amdgpu_device *adev)
 960 {
 961         return amdgpu_bo_create_kernel(adev, AMDGPU_GPU_PAGE_SIZE, PAGE_SIZE,
 962                                        AMDGPU_GEM_DOMAIN_VRAM |
 963                                        AMDGPU_GEM_DOMAIN_GTT,
 964                                        &adev->mem_scratch.robj,
 965                                        &adev->mem_scratch.gpu_addr,
 966                                        (void **)&adev->mem_scratch.ptr);
 967 }
 968
 969 /**
 970  * amdgpu_device_mem_scratch_fini - Free the VRAM scratch page
 971  *
 972  * @adev: amdgpu_device pointer
 973  *
 974  * Frees the VRAM scratch page.
 975  */
 976 static void amdgpu_device_mem_scratch_fini(struct amdgpu_device *adev)
 977 {
 978         amdgpu_bo_free_kernel(&adev->mem_scratch.robj, NULL, NULL);
 979 }
 980
 981 /**
 982  * amdgpu_device_program_register_sequence - program an array of registers.
 983  *
 984  * @adev: amdgpu_device pointer
 985  * @registers: pointer to the register array
 986  * @array_size: size of the register array
 987  *
 988  * Programs an array or registers with and and or masks.
 989  * This is a helper for setting golden registers.
 990  */
 991 void amdgpu_device_program_register_sequence(struct amdgpu_device *adev,
 992                                              const u32 *registers,
 993                                              const u32 array_size)
 994 {
 995         u32 tmp, reg, and_mask, or_mask;
 996         int i;
 997
 998         if (array_size % 3)
 999                 return;
1000
1001         for (i = 0; i < array_size; i +=3) {
1002                 reg = registers[i + 0];
1003                 and_mask = registers[i + 1];
1004                 or_mask = registers[i + 2];
1005
1006                 if (and_mask == 0xffffffff) {
1007                         tmp = or_mask;
1008                 } else {
1009                         tmp = RREG32(reg);
1010                         tmp &= ~and_mask;
1011                         if (adev->family >= AMDGPU_FAMILY_AI)
1012                                 tmp |= (or_mask & and_mask);
1013                         else
1014                                 tmp |= or_mask;
1015                 }
1016                 WREG32(reg, tmp);
1017         }
1018 }
1019
1020 /**
1021  * amdgpu_device_pci_config_reset - reset the GPU
1022  *
1023  * @adev: amdgpu_device pointer
1024  *
1025  * Resets the GPU using the pci config reset sequence.
1026  * Only applicable to asics prior to vega10.
1027  */
1028 void amdgpu_device_pci_config_reset(struct amdgpu_device *adev)
1029 {
1030         pci_write_config_dword(adev->pdev, 0x7c, AMDGPU_ASIC_RESET_DATA);
1031 }
1032
1033 /**
1034  * amdgpu_device_pci_reset - reset the GPU using generic PCI means
1035  *
1036  * @adev: amdgpu_device pointer
1037  *
1038  * Resets the GPU using generic pci reset interfaces (FLR, SBR, etc.).
1039  */
1040 int amdgpu_device_pci_reset(struct amdgpu_device *adev)
1041 {
1042         return pci_reset_function(adev->pdev);
1043 }
1044
1045 /*
1046  * GPU doorbell aperture helpers function.
1047  */
1048 /**
1049  * amdgpu_device_doorbell_init - Init doorbell driver information.
1050  *
1051  * @adev: amdgpu_device pointer
1052  *
1053  * Init doorbell driver information (CIK)
1054  * Returns 0 on success, error on failure.
1055  */
1056 static int amdgpu_device_doorbell_init(struct amdgpu_device *adev)
1057 {
1058
1059         /* No doorbell on SI hardware generation */
1060         if (adev->asic_type < CHIP_BONAIRE) {
1061                 adev->doorbell.base = 0;
1062                 adev->doorbell.size = 0;
1063                 adev->doorbell.num_kernel_doorbells = 0;
1064                 adev->doorbell.ptr = NULL;
1065                 return 0;
1066         }
1067
1068         if (pci_resource_flags(adev->pdev, 2) & IORESOURCE_UNSET)
1069                 return -EINVAL;
1070
1071         amdgpu_asic_init_doorbell_index(adev);
1072
1073         /* doorbell bar mapping */
1074         adev->doorbell.base = pci_resource_start(adev->pdev, 2);
1075         adev->doorbell.size = pci_resource_len(adev->pdev, 2);
1076
1077         if (adev->enable_mes) {
1078                 adev->doorbell.num_kernel_doorbells =
1079                         adev->doorbell.size / sizeof(u32);
1080         } else {
1081                 adev->doorbell.num_kernel_doorbells =
1082                         min_t(u32, adev->doorbell.size / sizeof(u32),
1083                               adev->doorbell_index.max_assignment+1);
1084                 if (adev->doorbell.num_kernel_doorbells == 0)
1085                         return -EINVAL;
1086
1087                 /* For Vega, reserve and map two pages on doorbell BAR since SDMA
1088                  * paging queue doorbell use the second page. The
1089                  * AMDGPU_DOORBELL64_MAX_ASSIGNMENT definition assumes all the
1090                  * doorbells are in the first page. So with paging queue enabled,
1091                  * the max num_kernel_doorbells should + 1 page (0x400 in dword)
1092                  */
1093                 if (adev->asic_type >= CHIP_VEGA10)
1094                         adev->doorbell.num_kernel_doorbells += 0x400;
1095         }
1096
1097         adev->doorbell.ptr = ioremap(adev->doorbell.base,
1098                                      adev->doorbell.num_kernel_doorbells *
1099                                      sizeof(u32));
1100         if (adev->doorbell.ptr == NULL)
1101                 return -ENOMEM;
1102
1103         return 0;
1104 }
1105
1106 /**
1107  * amdgpu_device_doorbell_fini - Tear down doorbell driver information.
1108  *
1109  * @adev: amdgpu_device pointer
1110  *
1111  * Tear down doorbell driver information (CIK)
1112  */
1113 static void amdgpu_device_doorbell_fini(struct amdgpu_device *adev)
1114 {
1115         iounmap(adev->doorbell.ptr);
1116         adev->doorbell.ptr = NULL;
1117 }
1118
1119
1120
1121 /*
1122  * amdgpu_device_wb_*()
1123  * Writeback is the method by which the GPU updates special pages in memory
1124  * with the status of certain GPU events (fences, ring pointers,etc.).
1125  */
1126
1127 /**
1128  * amdgpu_device_wb_fini - Disable Writeback and free memory
1129  *
1130  * @adev: amdgpu_device pointer
1131  *
1132  * Disables Writeback and frees the Writeback memory (all asics).
1133  * Used at driver shutdown.
1134  */
1135 static void amdgpu_device_wb_fini(struct amdgpu_device *adev)
1136 {
1137         if (adev->wb.wb_obj) {
1138                 amdgpu_bo_free_kernel(&adev->wb.wb_obj,
1139                                       &adev->wb.gpu_addr,
1140                                       (void **)&adev->wb.wb);
1141                 adev->wb.wb_obj = NULL;
1142         }
1143 }
1144
1145 /**
1146  * amdgpu_device_wb_init - Init Writeback driver info and allocate memory
1147  *
1148  * @adev: amdgpu_device pointer
1149  *
1150  * Initializes writeback and allocates writeback memory (all asics).
1151  * Used at driver startup.
1152  * Returns 0 on success or an -error on failure.
1153  */
1154 static int amdgpu_device_wb_init(struct amdgpu_device *adev)
1155 {
1156         int r;
1157
1158         if (adev->wb.wb_obj == NULL) {
1159                 /* AMDGPU_MAX_WB * sizeof(uint32_t) * 8 = AMDGPU_MAX_WB 256bit slots */
1160                 r = amdgpu_bo_create_kernel(adev, AMDGPU_MAX_WB * sizeof(uint32_t) * 8,
1161                                             PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
1162                                             &adev->wb.wb_obj, &adev->wb.gpu_addr,
1163                                             (void **)&adev->wb.wb);
1164                 if (r) {
1165                         dev_warn(adev->dev, "(%d) create WB bo failed\n", r);
1166                         return r;
1167                 }
1168
1169                 adev->wb.num_wb = AMDGPU_MAX_WB;
1170                 memset(&adev->wb.used, 0, sizeof(adev->wb.used));
1171
1172                 /* clear wb memory */
1173                 memset((char *)adev->wb.wb, 0, AMDGPU_MAX_WB * sizeof(uint32_t) * 8);
1174         }
1175
1176         return 0;
1177 }
1178
1179 /**
1180  * amdgpu_device_wb_get - Allocate a wb entry
1181  *
1182  * @adev: amdgpu_device pointer
1183  * @wb: wb index
1184  *
1185  * Allocate a wb slot for use by the driver (all asics).
1186  * Returns 0 on success or -EINVAL on failure.
1187  */
1188 int amdgpu_device_wb_get(struct amdgpu_device *adev, u32 *wb)
1189 {
1190         unsigned long offset = find_first_zero_bit(adev->wb.used, adev->wb.num_wb);
1191
1192         if (offset < adev->wb.num_wb) {
1193                 __set_bit(offset, adev->wb.used);
1194                 *wb = offset << 3; /* convert to dw offset */
1195                 return 0;
1196         } else {
1197                 return -EINVAL;
1198         }
1199 }
1200
1201 /**
1202  * amdgpu_device_wb_free - Free a wb entry
1203  *
1204  * @adev: amdgpu_device pointer
1205  * @wb: wb index
1206  *
1207  * Free a wb slot allocated for use by the driver (all asics)
1208  */
1209 void amdgpu_device_wb_free(struct amdgpu_device *adev, u32 wb)
1210 {
1211         wb >>= 3;
1212         if (wb < adev->wb.num_wb)
1213                 __clear_bit(wb, adev->wb.used);
1214 }
1215
1216 /**
1217  * amdgpu_device_resize_fb_bar - try to resize FB BAR
1218  *
1219  * @adev: amdgpu_device pointer
1220  *
1221  * Try to resize FB BAR to make all VRAM CPU accessible. We try very hard not
1222  * to fail, but if any of the BARs is not accessible after the size we abort
1223  * driver loading by returning -ENODEV.
1224  */
1225 int amdgpu_device_resize_fb_bar(struct amdgpu_device *adev)
1226 {
1227         int rbar_size = pci_rebar_bytes_to_size(adev->gmc.real_vram_size);
1228         struct pci_bus *root;
1229         struct resource *res;
1230         unsigned i;
1231         u16 cmd;
1232         int r;
1233
1234         /* Bypass for VF */
1235         if (amdgpu_sriov_vf(adev))
1236                 return 0;
1237
1238         /* skip if the bios has already enabled large BAR */
1239         if (adev->gmc.real_vram_size &&
1240             (pci_resource_len(adev->pdev, 0) >= adev->gmc.real_vram_size))
1241                 return 0;
1242
1243         /* Check if the root BUS has 64bit memory resources */
1244         root = adev->pdev->bus;
1245         while (root->parent)
1246                 root = root->parent;
1247
1248         pci_bus_for_each_resource(root, res, i) {
1249                 if (res && res->flags & (IORESOURCE_MEM | IORESOURCE_MEM_64) &&
1250                     res->start > 0x100000000ull)
1251                         break;
1252         }
1253
1254         /* Trying to resize is pointless without a root hub window above 4GB */
1255         if (!res)
1256                 return 0;
1257
1258         /* Limit the BAR size to what is available */
1259         rbar_size = min(fls(pci_rebar_get_possible_sizes(adev->pdev, 0)) - 1,
1260                         rbar_size);
1261
1262         /* Disable memory decoding while we change the BAR addresses and size */
1263         pci_read_config_word(adev->pdev, PCI_COMMAND, &cmd);
1264         pci_write_config_word(adev->pdev, PCI_COMMAND,
1265                               cmd & ~PCI_COMMAND_MEMORY);
1266
1267         /* Free the VRAM and doorbell BAR, we most likely need to move both. */
1268         amdgpu_device_doorbell_fini(adev);
1269         if (adev->asic_type >= CHIP_BONAIRE)
1270                 pci_release_resource(adev->pdev, 2);
1271
1272         pci_release_resource(adev->pdev, 0);
1273
1274         r = pci_resize_resource(adev->pdev, 0, rbar_size);
1275         if (r == -ENOSPC)
1276                 DRM_INFO("Not enough PCI address space for a large BAR.");
1277         else if (r && r != -ENOTSUPP)
1278                 DRM_ERROR("Problem resizing BAR0 (%d).", r);
1279
1280         pci_assign_unassigned_bus_resources(adev->pdev->bus);
1281
1282         /* When the doorbell or fb BAR isn't available we have no chance of
1283          * using the device.
1284          */
1285         r = amdgpu_device_doorbell_init(adev);
1286         if (r || (pci_resource_flags(adev->pdev, 0) & IORESOURCE_UNSET))
1287                 return -ENODEV;
1288
1289         pci_write_config_word(adev->pdev, PCI_COMMAND, cmd);
1290
1291         return 0;
1292 }
1293
1294 /*
1295  * GPU helpers function.
1296  */
1297 /**
1298  * amdgpu_device_need_post - check if the hw need post or not
1299  *
1300  * @adev: amdgpu_device pointer
1301  *
1302  * Check if the asic has been initialized (all asics) at driver startup
1303  * or post is needed if  hw reset is performed.
1304  * Returns true if need or false if not.
1305  */
1306 bool amdgpu_device_need_post(struct amdgpu_device *adev)
1307 {
1308         uint32_t reg;
1309
1310         if (amdgpu_sriov_vf(adev))
1311                 return false;
1312
1313         if (amdgpu_passthrough(adev)) {
1314                 /* for FIJI: In whole GPU pass-through virtualization case, after VM reboot
1315                  * some old smc fw still need driver do vPost otherwise gpu hang, while
1316                  * those smc fw version above 22.15 doesn't have this flaw, so we force
1317                  * vpost executed for smc version below 22.15
1318                  */
1319                 if (adev->asic_type == CHIP_FIJI) {
1320                         int err;
1321                         uint32_t fw_ver;
1322                         err = request_firmware(&adev->pm.fw, "amdgpu/fiji_smc.bin", adev->dev);
1323                         /* force vPost if error occured */
1324                         if (err)
1325                                 return true;
1326
1327                         fw_ver = *((uint32_t *)adev->pm.fw->data + 69);
1328                         if (fw_ver < 0x00160e00)
1329                                 return true;
1330                 }
1331         }
1332
1333         /* Don't post if we need to reset whole hive on init */
1334         if (adev->gmc.xgmi.pending_reset)
1335                 return false;
1336
1337         if (adev->has_hw_reset) {
1338                 adev->has_hw_reset = false;
1339                 return true;
1340         }
1341
1342         /* bios scratch used on CIK+ */
1343         if (adev->asic_type >= CHIP_BONAIRE)
1344                 return amdgpu_atombios_scratch_need_asic_init(adev);
1345
1346         /* check MEM_SIZE for older asics */
1347         reg = amdgpu_asic_get_config_memsize(adev);
1348
1349         if ((reg != 0) && (reg != 0xffffffff))
1350                 return false;
1351
1352         return true;
1353 }
1354
1355 /**
1356  * amdgpu_device_should_use_aspm - check if the device should program ASPM
1357  *
1358  * @adev: amdgpu_device pointer
1359  *
1360  * Confirm whether the module parameter and pcie bridge agree that ASPM should
1361  * be set for this device.
1362  *
1363  * Returns true if it should be used or false if not.
1364  */
1365 bool amdgpu_device_should_use_aspm(struct amdgpu_device *adev)
1366 {
1367         switch (amdgpu_aspm) {
1368         case -1:
1369                 break;
1370         case 0:
1371                 return false;
1372         case 1:
1373                 return true;
1374         default:
1375                 return false;
1376         }
1377         return pcie_aspm_enabled(adev->pdev);
1378 }
1379
1380 bool amdgpu_device_aspm_support_quirk(void)
1381 {
1382 #if IS_ENABLED(CONFIG_X86)
1383         struct cpuinfo_x86 *c = &cpu_data(0);
1384
1385         return !(c->x86 == 6 && c->x86_model == INTEL_FAM6_ALDERLAKE);
1386 #else
1387         return true;
1388 #endif
1389 }
1390
1391 /* if we get transitioned to only one device, take VGA back */
1392 /**
1393  * amdgpu_device_vga_set_decode - enable/disable vga decode
1394  *
1395  * @pdev: PCI device pointer
1396  * @state: enable/disable vga decode
1397  *
1398  * Enable/disable vga decode (all asics).
1399  * Returns VGA resource flags.
1400  */
1401 static unsigned int amdgpu_device_vga_set_decode(struct pci_dev *pdev,
1402                 bool state)
1403 {
1404         struct amdgpu_device *adev = drm_to_adev(pci_get_drvdata(pdev));
1405         amdgpu_asic_set_vga_state(adev, state);
1406         if (state)
1407                 return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM |
1408                        VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
1409         else
1410                 return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
1411 }
1412
1413 /**
1414  * amdgpu_device_check_block_size - validate the vm block size
1415  *
1416  * @adev: amdgpu_device pointer
1417  *
1418  * Validates the vm block size specified via module parameter.
1419  * The vm block size defines number of bits in page table versus page directory,
1420  * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
1421  * page table and the remaining bits are in the page directory.
1422  */
1423 static void amdgpu_device_check_block_size(struct amdgpu_device *adev)
1424 {
1425         /* defines number of bits in page table versus page directory,
1426          * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
1427          * page table and the remaining bits are in the page directory */
1428         if (amdgpu_vm_block_size == -1)
1429                 return;
1430
1431         if (amdgpu_vm_block_size < 9) {
1432                 dev_warn(adev->dev, "VM page table size (%d) too small\n",
1433                          amdgpu_vm_block_size);
1434                 amdgpu_vm_block_size = -1;
1435         }
1436 }
1437
1438 /**
1439  * amdgpu_device_check_vm_size - validate the vm size
1440  *
1441  * @adev: amdgpu_device pointer
1442  *
1443  * Validates the vm size in GB specified via module parameter.
1444  * The VM size is the size of the GPU virtual memory space in GB.
1445  */
1446 static void amdgpu_device_check_vm_size(struct amdgpu_device *adev)
1447 {
1448         /* no need to check the default value */
1449         if (amdgpu_vm_size == -1)
1450                 return;
1451
1452         if (amdgpu_vm_size < 1) {
1453                 dev_warn(adev->dev, "VM size (%d) too small, min is 1GB\n",
1454                          amdgpu_vm_size);
1455                 amdgpu_vm_size = -1;
1456         }
1457 }
1458
1459 static void amdgpu_device_check_smu_prv_buffer_size(struct amdgpu_device *adev)
1460 {
1461         struct sysinfo si;
1462         bool is_os_64 = (sizeof(void *) == 8);
1463         uint64_t total_memory;
1464         uint64_t dram_size_seven_GB = 0x1B8000000;
1465         uint64_t dram_size_three_GB = 0xB8000000;
1466
1467         if (amdgpu_smu_memory_pool_size == 0)
1468                 return;
1469
1470         if (!is_os_64) {
1471                 DRM_WARN("Not 64-bit OS, feature not supported\n");
1472                 goto def_value;
1473         }
1474         si_meminfo(&si);
1475         total_memory = (uint64_t)si.totalram * si.mem_unit;
1476
1477         if ((amdgpu_smu_memory_pool_size == 1) ||
1478                 (amdgpu_smu_memory_pool_size == 2)) {
1479                 if (total_memory < dram_size_three_GB)
1480                         goto def_value1;
1481         } else if ((amdgpu_smu_memory_pool_size == 4) ||
1482                 (amdgpu_smu_memory_pool_size == 8)) {
1483                 if (total_memory < dram_size_seven_GB)
1484                         goto def_value1;
1485         } else {
1486                 DRM_WARN("Smu memory pool size not supported\n");
1487                 goto def_value;
1488         }
1489         adev->pm.smu_prv_buffer_size = amdgpu_smu_memory_pool_size << 28;
1490
1491         return;
1492
1493 def_value1:
1494         DRM_WARN("No enough system memory\n");
1495 def_value:
1496         adev->pm.smu_prv_buffer_size = 0;
1497 }
1498
1499 static int amdgpu_device_init_apu_flags(struct amdgpu_device *adev)
1500 {
1501         if (!(adev->flags & AMD_IS_APU) ||
1502             adev->asic_type < CHIP_RAVEN)
1503                 return 0;
1504
1505         switch (adev->asic_type) {
1506         case CHIP_RAVEN:
1507                 if (adev->pdev->device == 0x15dd)
1508                         adev->apu_flags |= AMD_APU_IS_RAVEN;
1509                 if (adev->pdev->device == 0x15d8)
1510                         adev->apu_flags |= AMD_APU_IS_PICASSO;
1511                 break;
1512         case CHIP_RENOIR:
1513                 if ((adev->pdev->device == 0x1636) ||
1514                     (adev->pdev->device == 0x164c))
1515                         adev->apu_flags |= AMD_APU_IS_RENOIR;
1516                 else
1517                         adev->apu_flags |= AMD_APU_IS_GREEN_SARDINE;
1518                 break;
1519         case CHIP_VANGOGH:
1520                 adev->apu_flags |= AMD_APU_IS_VANGOGH;
1521                 break;
1522         case CHIP_YELLOW_CARP:
1523                 break;
1524         case CHIP_CYAN_SKILLFISH:
1525                 if ((adev->pdev->device == 0x13FE) ||
1526                     (adev->pdev->device == 0x143F))
1527                         adev->apu_flags |= AMD_APU_IS_CYAN_SKILLFISH2;
1528                 break;
1529         default:
1530                 break;
1531         }
1532
1533         return 0;
1534 }
1535
1536 /**
1537  * amdgpu_device_check_arguments - validate module params
1538  *
1539  * @adev: amdgpu_device pointer
1540  *
1541  * Validates certain module parameters and updates
1542  * the associated values used by the driver (all asics).
1543  */
1544 static int amdgpu_device_check_arguments(struct amdgpu_device *adev)
1545 {
1546         if (amdgpu_sched_jobs < 4) {
1547                 dev_warn(adev->dev, "sched jobs (%d) must be at least 4\n",
1548                          amdgpu_sched_jobs);
1549                 amdgpu_sched_jobs = 4;
1550         } else if (!is_power_of_2(amdgpu_sched_jobs)){
1551                 dev_warn(adev->dev, "sched jobs (%d) must be a power of 2\n",
1552                          amdgpu_sched_jobs);
1553                 amdgpu_sched_jobs = roundup_pow_of_two(amdgpu_sched_jobs);
1554         }
1555
1556         if (amdgpu_gart_size != -1 && amdgpu_gart_size < 32) {
1557                 /* gart size must be greater or equal to 32M */
1558                 dev_warn(adev->dev, "gart size (%d) too small\n",
1559                          amdgpu_gart_size);
1560                 amdgpu_gart_size = -1;
1561         }
1562
1563         if (amdgpu_gtt_size != -1 && amdgpu_gtt_size < 32) {
1564                 /* gtt size must be greater or equal to 32M */
1565                 dev_warn(adev->dev, "gtt size (%d) too small\n",
1566                                  amdgpu_gtt_size);
1567                 amdgpu_gtt_size = -1;
1568         }
1569
1570         /* valid range is between 4 and 9 inclusive */
1571         if (amdgpu_vm_fragment_size != -1 &&
1572             (amdgpu_vm_fragment_size > 9 || amdgpu_vm_fragment_size < 4)) {
1573                 dev_warn(adev->dev, "valid range is between 4 and 9\n");
1574                 amdgpu_vm_fragment_size = -1;
1575         }
1576
1577         if (amdgpu_sched_hw_submission < 2) {
1578                 dev_warn(adev->dev, "sched hw submission jobs (%d) must be at least 2\n",
1579                          amdgpu_sched_hw_submission);
1580                 amdgpu_sched_hw_submission = 2;
1581         } else if (!is_power_of_2(amdgpu_sched_hw_submission)) {
1582                 dev_warn(adev->dev, "sched hw submission jobs (%d) must be a power of 2\n",
1583                          amdgpu_sched_hw_submission);
1584                 amdgpu_sched_hw_submission = roundup_pow_of_two(amdgpu_sched_hw_submission);
1585         }
1586
1587         if (amdgpu_reset_method < -1 || amdgpu_reset_method > 4) {
1588                 dev_warn(adev->dev, "invalid option for reset method, reverting to default\n");
1589                 amdgpu_reset_method = -1;
1590         }
1591
1592         amdgpu_device_check_smu_prv_buffer_size(adev);
1593
1594         amdgpu_device_check_vm_size(adev);
1595
1596         amdgpu_device_check_block_size(adev);
1597
1598         adev->firmware.load_type = amdgpu_ucode_get_load_type(adev, amdgpu_fw_load_type);
1599
1600         return 0;
1601 }
1602
1603 /**
1604  * amdgpu_switcheroo_set_state - set switcheroo state
1605  *
1606  * @pdev: pci dev pointer
1607  * @state: vga_switcheroo state
1608  *
1609  * Callback for the switcheroo driver.  Suspends or resumes
1610  * the asics before or after it is powered up using ACPI methods.
1611  */
1612 static void amdgpu_switcheroo_set_state(struct pci_dev *pdev,
1613                                         enum vga_switcheroo_state state)
1614 {
1615         struct drm_device *dev = pci_get_drvdata(pdev);
1616         int r;
1617
1618         if (amdgpu_device_supports_px(dev) && state == VGA_SWITCHEROO_OFF)
1619                 return;
1620
1621         if (state == VGA_SWITCHEROO_ON) {
1622                 pr_info("switched on\n");
1623                 /* don't suspend or resume card normally */
1624                 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1625
1626                 pci_set_power_state(pdev, PCI_D0);
1627                 amdgpu_device_load_pci_state(pdev);
1628                 r = pci_enable_device(pdev);
1629                 if (r)
1630                         DRM_WARN("pci_enable_device failed (%d)\n", r);
1631                 amdgpu_device_resume(dev, true);
1632
1633                 dev->switch_power_state = DRM_SWITCH_POWER_ON;
1634         } else {
1635                 pr_info("switched off\n");
1636                 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1637                 amdgpu_device_suspend(dev, true);
1638                 amdgpu_device_cache_pci_state(pdev);
1639                 /* Shut down the device */
1640                 pci_disable_device(pdev);
1641                 pci_set_power_state(pdev, PCI_D3cold);
1642                 dev->switch_power_state = DRM_SWITCH_POWER_OFF;
1643         }
1644 }
1645
1646 /**
1647  * amdgpu_switcheroo_can_switch - see if switcheroo state can change
1648  *
1649  * @pdev: pci dev pointer
1650  *
1651  * Callback for the switcheroo driver.  Check of the switcheroo
1652  * state can be changed.
1653  * Returns true if the state can be changed, false if not.
1654  */
1655 static bool amdgpu_switcheroo_can_switch(struct pci_dev *pdev)
1656 {
1657         struct drm_device *dev = pci_get_drvdata(pdev);
1658
1659         /*
1660         * FIXME: open_count is protected by drm_global_mutex but that would lead to
1661         * locking inversion with the driver load path. And the access here is
1662         * completely racy anyway. So don't bother with locking for now.
1663         */
1664         return atomic_read(&dev->open_count) == 0;
1665 }
1666
1667 static const struct vga_switcheroo_client_ops amdgpu_switcheroo_ops = {
1668         .set_gpu_state = amdgpu_switcheroo_set_state,
1669         .reprobe = NULL,
1670         .can_switch = amdgpu_switcheroo_can_switch,
1671 };
1672
1673 /**
1674  * amdgpu_device_ip_set_clockgating_state - set the CG state
1675  *
1676  * @dev: amdgpu_device pointer
1677  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1678  * @state: clockgating state (gate or ungate)
1679  *
1680  * Sets the requested clockgating state for all instances of
1681  * the hardware IP specified.
1682  * Returns the error code from the last instance.
1683  */
1684 int amdgpu_device_ip_set_clockgating_state(void *dev,
1685                                            enum amd_ip_block_type block_type,
1686                                            enum amd_clockgating_state state)
1687 {
1688         struct amdgpu_device *adev = dev;
1689         int i, r = 0;
1690
1691         for (i = 0; i < adev->num_ip_blocks; i++) {
1692                 if (!adev->ip_blocks[i].status.valid)
1693                         continue;
1694                 if (adev->ip_blocks[i].version->type != block_type)
1695                         continue;
1696                 if (!adev->ip_blocks[i].version->funcs->set_clockgating_state)
1697                         continue;
1698                 r = adev->ip_blocks[i].version->funcs->set_clockgating_state(
1699                         (void *)adev, state);
1700                 if (r)
1701                         DRM_ERROR("set_clockgating_state of IP block <%s> failed %d\n",
1702                                   adev->ip_blocks[i].version->funcs->name, r);
1703         }
1704         return r;
1705 }
1706
1707 /**
1708  * amdgpu_device_ip_set_powergating_state - set the PG state
1709  *
1710  * @dev: amdgpu_device pointer
1711  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1712  * @state: powergating state (gate or ungate)
1713  *
1714  * Sets the requested powergating state for all instances of
1715  * the hardware IP specified.
1716  * Returns the error code from the last instance.
1717  */
1718 int amdgpu_device_ip_set_powergating_state(void *dev,
1719                                            enum amd_ip_block_type block_type,
1720                                            enum amd_powergating_state state)
1721 {
1722         struct amdgpu_device *adev = dev;
1723         int i, r = 0;
1724
1725         for (i = 0; i < adev->num_ip_blocks; i++) {
1726                 if (!adev->ip_blocks[i].status.valid)
1727                         continue;
1728                 if (adev->ip_blocks[i].version->type != block_type)
1729                         continue;
1730                 if (!adev->ip_blocks[i].version->funcs->set_powergating_state)
1731                         continue;
1732                 r = adev->ip_blocks[i].version->funcs->set_powergating_state(
1733                         (void *)adev, state);
1734                 if (r)
1735                         DRM_ERROR("set_powergating_state of IP block <%s> failed %d\n",
1736                                   adev->ip_blocks[i].version->funcs->name, r);
1737         }
1738         return r;
1739 }
1740
1741 /**
1742  * amdgpu_device_ip_get_clockgating_state - get the CG state
1743  *
1744  * @adev: amdgpu_device pointer
1745  * @flags: clockgating feature flags
1746  *
1747  * Walks the list of IPs on the device and updates the clockgating
1748  * flags for each IP.
1749  * Updates @flags with the feature flags for each hardware IP where
1750  * clockgating is enabled.
1751  */
1752 void amdgpu_device_ip_get_clockgating_state(struct amdgpu_device *adev,
1753                                             u64 *flags)
1754 {
1755         int i;
1756
1757         for (i = 0; i < adev->num_ip_blocks; i++) {
1758                 if (!adev->ip_blocks[i].status.valid)
1759                         continue;
1760                 if (adev->ip_blocks[i].version->funcs->get_clockgating_state)
1761                         adev->ip_blocks[i].version->funcs->get_clockgating_state((void *)adev, flags);
1762         }
1763 }
1764
1765 /**
1766  * amdgpu_device_ip_wait_for_idle - wait for idle
1767  *
1768  * @adev: amdgpu_device pointer
1769  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1770  *
1771  * Waits for the request hardware IP to be idle.
1772  * Returns 0 for success or a negative error code on failure.
1773  */
1774 int amdgpu_device_ip_wait_for_idle(struct amdgpu_device *adev,
1775                                    enum amd_ip_block_type block_type)
1776 {
1777         int i, r;
1778
1779         for (i = 0; i < adev->num_ip_blocks; i++) {
1780                 if (!adev->ip_blocks[i].status.valid)
1781                         continue;
1782                 if (adev->ip_blocks[i].version->type == block_type) {
1783                         r = adev->ip_blocks[i].version->funcs->wait_for_idle((void *)adev);
1784                         if (r)
1785                                 return r;
1786                         break;
1787                 }
1788         }
1789         return 0;
1790
1791 }
1792
1793 /**
1794  * amdgpu_device_ip_is_idle - is the hardware IP idle
1795  *
1796  * @adev: amdgpu_device pointer
1797  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1798  *
1799  * Check if the hardware IP is idle or not.
1800  * Returns true if it the IP is idle, false if not.
1801  */
1802 bool amdgpu_device_ip_is_idle(struct amdgpu_device *adev,
1803                               enum amd_ip_block_type block_type)
1804 {
1805         int i;
1806
1807         for (i = 0; i < adev->num_ip_blocks; i++) {
1808                 if (!adev->ip_blocks[i].status.valid)
1809                         continue;
1810                 if (adev->ip_blocks[i].version->type == block_type)
1811                         return adev->ip_blocks[i].version->funcs->is_idle((void *)adev);
1812         }
1813         return true;
1814
1815 }
1816
1817 /**
1818  * amdgpu_device_ip_get_ip_block - get a hw IP pointer
1819  *
1820  * @adev: amdgpu_device pointer
1821  * @type: Type of hardware IP (SMU, GFX, UVD, etc.)
1822  *
1823  * Returns a pointer to the hardware IP block structure
1824  * if it exists for the asic, otherwise NULL.
1825  */
1826 struct amdgpu_ip_block *
1827 amdgpu_device_ip_get_ip_block(struct amdgpu_device *adev,
1828                               enum amd_ip_block_type type)
1829 {
1830         int i;
1831
1832         for (i = 0; i < adev->num_ip_blocks; i++)
1833                 if (adev->ip_blocks[i].version->type == type)
1834                         return &adev->ip_blocks[i];
1835
1836         return NULL;
1837 }
1838
1839 /**
1840  * amdgpu_device_ip_block_version_cmp
1841  *
1842  * @adev: amdgpu_device pointer
1843  * @type: enum amd_ip_block_type
1844  * @major: major version
1845  * @minor: minor version
1846  *
1847  * return 0 if equal or greater
1848  * return 1 if smaller or the ip_block doesn't exist
1849  */
1850 int amdgpu_device_ip_block_version_cmp(struct amdgpu_device *adev,
1851                                        enum amd_ip_block_type type,
1852                                        u32 major, u32 minor)
1853 {
1854         struct amdgpu_ip_block *ip_block = amdgpu_device_ip_get_ip_block(adev, type);
1855
1856         if (ip_block && ((ip_block->version->major > major) ||
1857                         ((ip_block->version->major == major) &&
1858                         (ip_block->version->minor >= minor))))
1859                 return 0;
1860
1861         return 1;
1862 }
1863
1864 /**
1865  * amdgpu_device_ip_block_add
1866  *
1867  * @adev: amdgpu_device pointer
1868  * @ip_block_version: pointer to the IP to add
1869  *
1870  * Adds the IP block driver information to the collection of IPs
1871  * on the asic.
1872  */
1873 int amdgpu_device_ip_block_add(struct amdgpu_device *adev,
1874                                const struct amdgpu_ip_block_version *ip_block_version)
1875 {
1876         if (!ip_block_version)
1877                 return -EINVAL;
1878
1879         switch (ip_block_version->type) {
1880         case AMD_IP_BLOCK_TYPE_VCN:
1881                 if (adev->harvest_ip_mask & AMD_HARVEST_IP_VCN_MASK)
1882                         return 0;
1883                 break;
1884         case AMD_IP_BLOCK_TYPE_JPEG:
1885                 if (adev->harvest_ip_mask & AMD_HARVEST_IP_JPEG_MASK)
1886                         return 0;
1887                 break;
1888         default:
1889                 break;
1890         }
1891
1892         DRM_INFO("add ip block number %d <%s>\n", adev->num_ip_blocks,
1893                   ip_block_version->funcs->name);
1894
1895         adev->ip_blocks[adev->num_ip_blocks++].version = ip_block_version;
1896
1897         return 0;
1898 }
1899
1900 /**
1901  * amdgpu_device_enable_virtual_display - enable virtual display feature
1902  *
1903  * @adev: amdgpu_device pointer
1904  *
1905  * Enabled the virtual display feature if the user has enabled it via
1906  * the module parameter virtual_display.  This feature provides a virtual
1907  * display hardware on headless boards or in virtualized environments.
1908  * This function parses and validates the configuration string specified by
1909  * the user and configues the virtual display configuration (number of
1910  * virtual connectors, crtcs, etc.) specified.
1911  */
1912 static void amdgpu_device_enable_virtual_display(struct amdgpu_device *adev)
1913 {
1914         adev->enable_virtual_display = false;
1915
1916         if (amdgpu_virtual_display) {
1917                 const char *pci_address_name = pci_name(adev->pdev);
1918                 char *pciaddstr, *pciaddstr_tmp, *pciaddname_tmp, *pciaddname;
1919
1920                 pciaddstr = kstrdup(amdgpu_virtual_display, GFP_KERNEL);
1921                 pciaddstr_tmp = pciaddstr;
1922                 while ((pciaddname_tmp = strsep(&pciaddstr_tmp, ";"))) {
1923                         pciaddname = strsep(&pciaddname_tmp, ",");
1924                         if (!strcmp("all", pciaddname)
1925                             || !strcmp(pci_address_name, pciaddname)) {
1926                                 long num_crtc;
1927                                 int res = -1;
1928
1929                                 adev->enable_virtual_display = true;
1930
1931                                 if (pciaddname_tmp)
1932                                         res = kstrtol(pciaddname_tmp, 10,
1933                                                       &num_crtc);
1934
1935                                 if (!res) {
1936                                         if (num_crtc < 1)
1937                                                 num_crtc = 1;
1938                                         if (num_crtc > 6)
1939                                                 num_crtc = 6;
1940                                         adev->mode_info.num_crtc = num_crtc;
1941                                 } else {
1942                                         adev->mode_info.num_crtc = 1;
1943                                 }
1944                                 break;
1945                         }
1946                 }
1947
1948                 DRM_INFO("virtual display string:%s, %s:virtual_display:%d, num_crtc:%d\n",
1949                          amdgpu_virtual_display, pci_address_name,
1950                          adev->enable_virtual_display, adev->mode_info.num_crtc);
1951
1952                 kfree(pciaddstr);
1953         }
1954 }
1955
1956 void amdgpu_device_set_sriov_virtual_display(struct amdgpu_device *adev)
1957 {
1958         if (amdgpu_sriov_vf(adev) && !adev->enable_virtual_display) {
1959                 adev->mode_info.num_crtc = 1;
1960                 adev->enable_virtual_display = true;
1961                 DRM_INFO("virtual_display:%d, num_crtc:%d\n",
1962                          adev->enable_virtual_display, adev->mode_info.num_crtc);
1963         }
1964 }
1965
1966 /**
1967  * amdgpu_device_parse_gpu_info_fw - parse gpu info firmware
1968  *
1969  * @adev: amdgpu_device pointer
1970  *
1971  * Parses the asic configuration parameters specified in the gpu info
1972  * firmware and makes them availale to the driver for use in configuring
1973  * the asic.
1974  * Returns 0 on success, -EINVAL on failure.
1975  */
1976 static int amdgpu_device_parse_gpu_info_fw(struct amdgpu_device *adev)
1977 {
1978         const char *chip_name;
1979         char fw_name[40];
1980         int err;
1981         const struct gpu_info_firmware_header_v1_0 *hdr;
1982
1983         adev->firmware.gpu_info_fw = NULL;
1984
1985         if (adev->mman.discovery_bin) {
1986                 /*
1987                  * FIXME: The bounding box is still needed by Navi12, so
1988                  * temporarily read it from gpu_info firmware. Should be dropped
1989                  * when DAL no longer needs it.
1990                  */
1991                 if (adev->asic_type != CHIP_NAVI12)
1992                         return 0;
1993         }
1994
1995         switch (adev->asic_type) {
1996         default:
1997                 return 0;
1998         case CHIP_VEGA10:
1999                 chip_name = "vega10";
2000                 break;
2001         case CHIP_VEGA12:
2002                 chip_name = "vega12";
2003                 break;
2004         case CHIP_RAVEN:
2005                 if (adev->apu_flags & AMD_APU_IS_RAVEN2)
2006                         chip_name = "raven2";
2007                 else if (adev->apu_flags & AMD_APU_IS_PICASSO)
2008                         chip_name = "picasso";
2009                 else
2010                         chip_name = "raven";
2011                 break;
2012         case CHIP_ARCTURUS:
2013                 chip_name = "arcturus";
2014                 break;
2015         case CHIP_NAVI12:
2016                 chip_name = "navi12";
2017                 break;
2018         }
2019
2020         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_gpu_info.bin", chip_name);
2021         err = amdgpu_ucode_request(adev, &adev->firmware.gpu_info_fw, fw_name);
2022         if (err) {
2023                 dev_err(adev->dev,
2024                         "Failed to get gpu_info firmware \"%s\"\n",
2025                         fw_name);
2026                 goto out;
2027         }
2028
2029         hdr = (const struct gpu_info_firmware_header_v1_0 *)adev->firmware.gpu_info_fw->data;
2030         amdgpu_ucode_print_gpu_info_hdr(&hdr->header);
2031
2032         switch (hdr->version_major) {
2033         case 1:
2034         {
2035                 const struct gpu_info_firmware_v1_0 *gpu_info_fw =
2036                         (const struct gpu_info_firmware_v1_0 *)(adev->firmware.gpu_info_fw->data +
2037                                                                 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2038
2039                 /*
2040                  * Should be droped when DAL no longer needs it.
2041                  */
2042                 if (adev->asic_type == CHIP_NAVI12)
2043                         goto parse_soc_bounding_box;
2044
2045                 adev->gfx.config.max_shader_engines = le32_to_cpu(gpu_info_fw->gc_num_se);
2046                 adev->gfx.config.max_cu_per_sh = le32_to_cpu(gpu_info_fw->gc_num_cu_per_sh);
2047                 adev->gfx.config.max_sh_per_se = le32_to_cpu(gpu_info_fw->gc_num_sh_per_se);
2048                 adev->gfx.config.max_backends_per_se = le32_to_cpu(gpu_info_fw->gc_num_rb_per_se);
2049                 adev->gfx.config.max_texture_channel_caches =
2050                         le32_to_cpu(gpu_info_fw->gc_num_tccs);
2051                 adev->gfx.config.max_gprs = le32_to_cpu(gpu_info_fw->gc_num_gprs);
2052                 adev->gfx.config.max_gs_threads = le32_to_cpu(gpu_info_fw->gc_num_max_gs_thds);
2053                 adev->gfx.config.gs_vgt_table_depth = le32_to_cpu(gpu_info_fw->gc_gs_table_depth);
2054                 adev->gfx.config.gs_prim_buffer_depth = le32_to_cpu(gpu_info_fw->gc_gsprim_buff_depth);
2055                 adev->gfx.config.double_offchip_lds_buf =
2056                         le32_to_cpu(gpu_info_fw->gc_double_offchip_lds_buffer);
2057                 adev->gfx.cu_info.wave_front_size = le32_to_cpu(gpu_info_fw->gc_wave_size);
2058                 adev->gfx.cu_info.max_waves_per_simd =
2059                         le32_to_cpu(gpu_info_fw->gc_max_waves_per_simd);
2060                 adev->gfx.cu_info.max_scratch_slots_per_cu =
2061                         le32_to_cpu(gpu_info_fw->gc_max_scratch_slots_per_cu);
2062                 adev->gfx.cu_info.lds_size = le32_to_cpu(gpu_info_fw->gc_lds_size);
2063                 if (hdr->version_minor >= 1) {
2064                         const struct gpu_info_firmware_v1_1 *gpu_info_fw =
2065                                 (const struct gpu_info_firmware_v1_1 *)(adev->firmware.gpu_info_fw->data +
2066                                                                         le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2067                         adev->gfx.config.num_sc_per_sh =
2068                                 le32_to_cpu(gpu_info_fw->num_sc_per_sh);
2069                         adev->gfx.config.num_packer_per_sc =
2070                                 le32_to_cpu(gpu_info_fw->num_packer_per_sc);
2071                 }
2072
2073 parse_soc_bounding_box:
2074                 /*
2075                  * soc bounding box info is not integrated in disocovery table,
2076                  * we always need to parse it from gpu info firmware if needed.
2077                  */
2078                 if (hdr->version_minor == 2) {
2079                         const struct gpu_info_firmware_v1_2 *gpu_info_fw =
2080                                 (const struct gpu_info_firmware_v1_2 *)(adev->firmware.gpu_info_fw->data +
2081                                                                         le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2082                         adev->dm.soc_bounding_box = &gpu_info_fw->soc_bounding_box;
2083                 }
2084                 break;
2085         }
2086         default:
2087                 dev_err(adev->dev,
2088                         "Unsupported gpu_info table %d\n", hdr->header.ucode_version);
2089                 err = -EINVAL;
2090                 goto out;
2091         }
2092 out:
2093         return err;
2094 }
2095
2096 /**
2097  * amdgpu_device_ip_early_init - run early init for hardware IPs
2098  *
2099  * @adev: amdgpu_device pointer
2100  *
2101  * Early initialization pass for hardware IPs.  The hardware IPs that make
2102  * up each asic are discovered each IP's early_init callback is run.  This
2103  * is the first stage in initializing the asic.
2104  * Returns 0 on success, negative error code on failure.
2105  */
2106 static int amdgpu_device_ip_early_init(struct amdgpu_device *adev)
2107 {
2108         struct drm_device *dev = adev_to_drm(adev);
2109         struct pci_dev *parent;
2110         int i, r;
2111         bool total;
2112
2113         amdgpu_device_enable_virtual_display(adev);
2114
2115         if (amdgpu_sriov_vf(adev)) {
2116                 r = amdgpu_virt_request_full_gpu(adev, true);
2117                 if (r)
2118                         return r;
2119         }
2120
2121         switch (adev->asic_type) {
2122 #ifdef CONFIG_DRM_AMDGPU_SI
2123         case CHIP_VERDE:
2124         case CHIP_TAHITI:
2125         case CHIP_PITCAIRN:
2126         case CHIP_OLAND:
2127         case CHIP_HAINAN:
2128                 adev->family = AMDGPU_FAMILY_SI;
2129                 r = si_set_ip_blocks(adev);
2130                 if (r)
2131                         return r;
2132                 break;
2133 #endif
2134 #ifdef CONFIG_DRM_AMDGPU_CIK
2135         case CHIP_BONAIRE:
2136         case CHIP_HAWAII:
2137         case CHIP_KAVERI:
2138         case CHIP_KABINI:
2139         case CHIP_MULLINS:
2140                 if (adev->flags & AMD_IS_APU)
2141                         adev->family = AMDGPU_FAMILY_KV;
2142                 else
2143                         adev->family = AMDGPU_FAMILY_CI;
2144
2145                 r = cik_set_ip_blocks(adev);
2146                 if (r)
2147                         return r;
2148                 break;
2149 #endif
2150         case CHIP_TOPAZ:
2151         case CHIP_TONGA:
2152         case CHIP_FIJI:
2153         case CHIP_POLARIS10:
2154         case CHIP_POLARIS11:
2155         case CHIP_POLARIS12:
2156         case CHIP_VEGAM:
2157         case CHIP_CARRIZO:
2158         case CHIP_STONEY:
2159                 if (adev->flags & AMD_IS_APU)
2160                         adev->family = AMDGPU_FAMILY_CZ;
2161                 else
2162                         adev->family = AMDGPU_FAMILY_VI;
2163
2164                 r = vi_set_ip_blocks(adev);
2165                 if (r)
2166                         return r;
2167                 break;
2168         default:
2169                 r = amdgpu_discovery_set_ip_blocks(adev);
2170                 if (r)
2171                         return r;
2172                 break;
2173         }
2174
2175         if (amdgpu_has_atpx() &&
2176             (amdgpu_is_atpx_hybrid() ||
2177              amdgpu_has_atpx_dgpu_power_cntl()) &&
2178             ((adev->flags & AMD_IS_APU) == 0) &&
2179             !pci_is_thunderbolt_attached(to_pci_dev(dev->dev)))
2180                 adev->flags |= AMD_IS_PX;
2181
2182         if (!(adev->flags & AMD_IS_APU)) {
2183                 parent = pci_upstream_bridge(adev->pdev);
2184                 adev->has_pr3 = parent ? pci_pr3_present(parent) : false;
2185         }
2186
2187
2188         adev->pm.pp_feature = amdgpu_pp_feature_mask;
2189         if (amdgpu_sriov_vf(adev) || sched_policy == KFD_SCHED_POLICY_NO_HWS)
2190                 adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
2191         if (amdgpu_sriov_vf(adev) && adev->asic_type == CHIP_SIENNA_CICHLID)
2192                 adev->pm.pp_feature &= ~PP_OVERDRIVE_MASK;
2193
2194         total = true;
2195         for (i = 0; i < adev->num_ip_blocks; i++) {
2196                 if ((amdgpu_ip_block_mask & (1 << i)) == 0) {
2197                         DRM_ERROR("disabled ip block: %d <%s>\n",
2198                                   i, adev->ip_blocks[i].version->funcs->name);
2199                         adev->ip_blocks[i].status.valid = false;
2200                 } else {
2201                         if (adev->ip_blocks[i].version->funcs->early_init) {
2202                                 r = adev->ip_blocks[i].version->funcs->early_init((void *)adev);
2203                                 if (r == -ENOENT) {
2204                                         adev->ip_blocks[i].status.valid = false;
2205                                 } else if (r) {
2206                                         DRM_ERROR("early_init of IP block <%s> failed %d\n",
2207                                                   adev->ip_blocks[i].version->funcs->name, r);
2208                                         total = false;
2209                                 } else {
2210                                         adev->ip_blocks[i].status.valid = true;
2211                                 }
2212                         } else {
2213                                 adev->ip_blocks[i].status.valid = true;
2214                         }
2215                 }
2216                 /* get the vbios after the asic_funcs are set up */
2217                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) {
2218                         r = amdgpu_device_parse_gpu_info_fw(adev);
2219                         if (r)
2220                                 return r;
2221
2222                         /* Read BIOS */
2223                         if (!amdgpu_get_bios(adev))
2224                                 return -EINVAL;
2225
2226                         r = amdgpu_atombios_init(adev);
2227                         if (r) {
2228                                 dev_err(adev->dev, "amdgpu_atombios_init failed\n");
2229                                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_INIT_FAIL, 0, 0);
2230                                 return r;
2231                         }
2232
2233                         /*get pf2vf msg info at it's earliest time*/
2234                         if (amdgpu_sriov_vf(adev))
2235                                 amdgpu_virt_init_data_exchange(adev);
2236
2237                 }
2238         }
2239         if (!total)
2240                 return -ENODEV;
2241
2242         amdgpu_amdkfd_device_probe(adev);
2243         adev->cg_flags &= amdgpu_cg_mask;
2244         adev->pg_flags &= amdgpu_pg_mask;
2245
2246         return 0;
2247 }
2248
2249 static int amdgpu_device_ip_hw_init_phase1(struct amdgpu_device *adev)
2250 {
2251         int i, r;
2252
2253         for (i = 0; i < adev->num_ip_blocks; i++) {
2254                 if (!adev->ip_blocks[i].status.sw)
2255                         continue;
2256                 if (adev->ip_blocks[i].status.hw)
2257                         continue;
2258                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
2259                     (amdgpu_sriov_vf(adev) && (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)) ||
2260                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) {
2261                         r = adev->ip_blocks[i].version->funcs->hw_init(adev);
2262                         if (r) {
2263                                 DRM_ERROR("hw_init of IP block <%s> failed %d\n",
2264                                           adev->ip_blocks[i].version->funcs->name, r);
2265                                 return r;
2266                         }
2267                         adev->ip_blocks[i].status.hw = true;
2268                 }
2269         }
2270
2271         return 0;
2272 }
2273
2274 static int amdgpu_device_ip_hw_init_phase2(struct amdgpu_device *adev)
2275 {
2276         int i, r;
2277
2278         for (i = 0; i < adev->num_ip_blocks; i++) {
2279                 if (!adev->ip_blocks[i].status.sw)
2280                         continue;
2281                 if (adev->ip_blocks[i].status.hw)
2282                         continue;
2283                 r = adev->ip_blocks[i].version->funcs->hw_init(adev);
2284                 if (r) {
2285                         DRM_ERROR("hw_init of IP block <%s> failed %d\n",
2286                                   adev->ip_blocks[i].version->funcs->name, r);
2287                         return r;
2288                 }
2289                 adev->ip_blocks[i].status.hw = true;
2290         }
2291
2292         return 0;
2293 }
2294
2295 static int amdgpu_device_fw_loading(struct amdgpu_device *adev)
2296 {
2297         int r = 0;
2298         int i;
2299         uint32_t smu_version;
2300
2301         if (adev->asic_type >= CHIP_VEGA10) {
2302                 for (i = 0; i < adev->num_ip_blocks; i++) {
2303                         if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_PSP)
2304                                 continue;
2305
2306                         if (!adev->ip_blocks[i].status.sw)
2307                                 continue;
2308
2309                         /* no need to do the fw loading again if already done*/
2310                         if (adev->ip_blocks[i].status.hw == true)
2311                                 break;
2312
2313                         if (amdgpu_in_reset(adev) || adev->in_suspend) {
2314                                 r = adev->ip_blocks[i].version->funcs->resume(adev);
2315                                 if (r) {
2316                                         DRM_ERROR("resume of IP block <%s> failed %d\n",
2317                                                           adev->ip_blocks[i].version->funcs->name, r);
2318                                         return r;
2319                                 }
2320                         } else {
2321                                 r = adev->ip_blocks[i].version->funcs->hw_init(adev);
2322                                 if (r) {
2323                                         DRM_ERROR("hw_init of IP block <%s> failed %d\n",
2324                                                           adev->ip_blocks[i].version->funcs->name, r);
2325                                         return r;
2326                                 }
2327                         }
2328
2329                         adev->ip_blocks[i].status.hw = true;
2330                         break;
2331                 }
2332         }
2333
2334         if (!amdgpu_sriov_vf(adev) || adev->asic_type == CHIP_TONGA)
2335                 r = amdgpu_pm_load_smu_firmware(adev, &smu_version);
2336
2337         return r;
2338 }
2339
2340 static int amdgpu_device_init_schedulers(struct amdgpu_device *adev)
2341 {
2342         long timeout;
2343         int r, i;
2344
2345         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
2346                 struct amdgpu_ring *ring = adev->rings[i];
2347
2348                 /* No need to setup the GPU scheduler for rings that don't need it */
2349                 if (!ring || ring->no_scheduler)
2350                         continue;
2351
2352                 switch (ring->funcs->type) {
2353                 case AMDGPU_RING_TYPE_GFX:
2354                         timeout = adev->gfx_timeout;
2355                         break;
2356                 case AMDGPU_RING_TYPE_COMPUTE:
2357                         timeout = adev->compute_timeout;
2358                         break;
2359                 case AMDGPU_RING_TYPE_SDMA:
2360                         timeout = adev->sdma_timeout;
2361                         break;
2362                 default:
2363                         timeout = adev->video_timeout;
2364                         break;
2365                 }
2366
2367                 r = drm_sched_init(&ring->sched, &amdgpu_sched_ops,
2368                                    ring->num_hw_submission, 0,
2369                                    timeout, adev->reset_domain->wq,
2370                                    ring->sched_score, ring->name,
2371                                    adev->dev);
2372                 if (r) {
2373                         DRM_ERROR("Failed to create scheduler on ring %s.\n",
2374                                   ring->name);
2375                         return r;
2376                 }
2377         }
2378
2379         return 0;
2380 }
2381
2382
2383 /**
2384  * amdgpu_device_ip_init - run init for hardware IPs
2385  *
2386  * @adev: amdgpu_device pointer
2387  *
2388  * Main initialization pass for hardware IPs.  The list of all the hardware
2389  * IPs that make up the asic is walked and the sw_init and hw_init callbacks
2390  * are run.  sw_init initializes the software state associated with each IP
2391  * and hw_init initializes the hardware associated with each IP.
2392  * Returns 0 on success, negative error code on failure.
2393  */
2394 static int amdgpu_device_ip_init(struct amdgpu_device *adev)
2395 {
2396         int i, r;
2397
2398         r = amdgpu_ras_init(adev);
2399         if (r)
2400                 return r;
2401
2402         for (i = 0; i < adev->num_ip_blocks; i++) {
2403                 if (!adev->ip_blocks[i].status.valid)
2404                         continue;
2405                 r = adev->ip_blocks[i].version->funcs->sw_init((void *)adev);
2406                 if (r) {
2407                         DRM_ERROR("sw_init of IP block <%s> failed %d\n",
2408                                   adev->ip_blocks[i].version->funcs->name, r);
2409                         goto init_failed;
2410                 }
2411                 adev->ip_blocks[i].status.sw = true;
2412
2413                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) {
2414                         /* need to do common hw init early so everything is set up for gmc */
2415                         r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev);
2416                         if (r) {
2417                                 DRM_ERROR("hw_init %d failed %d\n", i, r);
2418                                 goto init_failed;
2419                         }
2420                         adev->ip_blocks[i].status.hw = true;
2421                 } else if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
2422                         /* need to do gmc hw init early so we can allocate gpu mem */
2423                         /* Try to reserve bad pages early */
2424                         if (amdgpu_sriov_vf(adev))
2425                                 amdgpu_virt_exchange_data(adev);
2426
2427                         r = amdgpu_device_mem_scratch_init(adev);
2428                         if (r) {
2429                                 DRM_ERROR("amdgpu_mem_scratch_init failed %d\n", r);
2430                                 goto init_failed;
2431                         }
2432                         r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev);
2433                         if (r) {
2434                                 DRM_ERROR("hw_init %d failed %d\n", i, r);
2435                                 goto init_failed;
2436                         }
2437                         r = amdgpu_device_wb_init(adev);
2438                         if (r) {
2439                                 DRM_ERROR("amdgpu_device_wb_init failed %d\n", r);
2440                                 goto init_failed;
2441                         }
2442                         adev->ip_blocks[i].status.hw = true;
2443
2444                         /* right after GMC hw init, we create CSA */
2445                         if (amdgpu_mcbp) {
2446                                 r = amdgpu_allocate_static_csa(adev, &adev->virt.csa_obj,
2447                                                                AMDGPU_GEM_DOMAIN_VRAM |
2448                                                                AMDGPU_GEM_DOMAIN_GTT,
2449                                                                AMDGPU_CSA_SIZE);
2450                                 if (r) {
2451                                         DRM_ERROR("allocate CSA failed %d\n", r);
2452                                         goto init_failed;
2453                                 }
2454                         }
2455                 }
2456         }
2457
2458         if (amdgpu_sriov_vf(adev))
2459                 amdgpu_virt_init_data_exchange(adev);
2460
2461         r = amdgpu_ib_pool_init(adev);
2462         if (r) {
2463                 dev_err(adev->dev, "IB initialization failed (%d).\n", r);
2464                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_IB_INIT_FAIL, 0, r);
2465                 goto init_failed;
2466         }
2467
2468         r = amdgpu_ucode_create_bo(adev); /* create ucode bo when sw_init complete*/
2469         if (r)
2470                 goto init_failed;
2471
2472         r = amdgpu_device_ip_hw_init_phase1(adev);
2473         if (r)
2474                 goto init_failed;
2475
2476         r = amdgpu_device_fw_loading(adev);
2477         if (r)
2478                 goto init_failed;
2479
2480         r = amdgpu_device_ip_hw_init_phase2(adev);
2481         if (r)
2482                 goto init_failed;
2483
2484         /*
2485          * retired pages will be loaded from eeprom and reserved here,
2486          * it should be called after amdgpu_device_ip_hw_init_phase2  since
2487          * for some ASICs the RAS EEPROM code relies on SMU fully functioning
2488          * for I2C communication which only true at this point.
2489          *
2490          * amdgpu_ras_recovery_init may fail, but the upper only cares the
2491          * failure from bad gpu situation and stop amdgpu init process
2492          * accordingly. For other failed cases, it will still release all
2493          * the resource and print error message, rather than returning one
2494          * negative value to upper level.
2495          *
2496          * Note: theoretically, this should be called before all vram allocations
2497          * to protect retired page from abusing
2498          */
2499         r = amdgpu_ras_recovery_init(adev);
2500         if (r)
2501                 goto init_failed;
2502
2503         /**
2504          * In case of XGMI grab extra reference for reset domain for this device
2505          */
2506         if (adev->gmc.xgmi.num_physical_nodes > 1) {
2507                 if (amdgpu_xgmi_add_device(adev) == 0) {
2508                         if (!amdgpu_sriov_vf(adev)) {
2509                                 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
2510
2511                                 if (WARN_ON(!hive)) {
2512                                         r = -ENOENT;
2513                                         goto init_failed;
2514                                 }
2515
2516                                 if (!hive->reset_domain ||
2517                                     !amdgpu_reset_get_reset_domain(hive->reset_domain)) {
2518                                         r = -ENOENT;
2519                                         amdgpu_put_xgmi_hive(hive);
2520                                         goto init_failed;
2521                                 }
2522
2523                                 /* Drop the early temporary reset domain we created for device */
2524                                 amdgpu_reset_put_reset_domain(adev->reset_domain);
2525                                 adev->reset_domain = hive->reset_domain;
2526                                 amdgpu_put_xgmi_hive(hive);
2527                         }
2528                 }
2529         }
2530
2531         r = amdgpu_device_init_schedulers(adev);
2532         if (r)
2533                 goto init_failed;
2534
2535         /* Don't init kfd if whole hive need to be reset during init */
2536         if (!adev->gmc.xgmi.pending_reset)
2537                 amdgpu_amdkfd_device_init(adev);
2538
2539         amdgpu_fru_get_product_info(adev);
2540
2541 init_failed:
2542         if (amdgpu_sriov_vf(adev))
2543                 amdgpu_virt_release_full_gpu(adev, true);
2544
2545         return r;
2546 }
2547
2548 /**
2549  * amdgpu_device_fill_reset_magic - writes reset magic to gart pointer
2550  *
2551  * @adev: amdgpu_device pointer
2552  *
2553  * Writes a reset magic value to the gart pointer in VRAM.  The driver calls
2554  * this function before a GPU reset.  If the value is retained after a
2555  * GPU reset, VRAM has not been lost.  Some GPU resets may destry VRAM contents.
2556  */
2557 static void amdgpu_device_fill_reset_magic(struct amdgpu_device *adev)
2558 {
2559         memcpy(adev->reset_magic, adev->gart.ptr, AMDGPU_RESET_MAGIC_NUM);
2560 }
2561
2562 /**
2563  * amdgpu_device_check_vram_lost - check if vram is valid
2564  *
2565  * @adev: amdgpu_device pointer
2566  *
2567  * Checks the reset magic value written to the gart pointer in VRAM.
2568  * The driver calls this after a GPU reset to see if the contents of
2569  * VRAM is lost or now.
2570  * returns true if vram is lost, false if not.
2571  */
2572 static bool amdgpu_device_check_vram_lost(struct amdgpu_device *adev)
2573 {
2574         if (memcmp(adev->gart.ptr, adev->reset_magic,
2575                         AMDGPU_RESET_MAGIC_NUM))
2576                 return true;
2577
2578         if (!amdgpu_in_reset(adev))
2579                 return false;
2580
2581         /*
2582          * For all ASICs with baco/mode1 reset, the VRAM is
2583          * always assumed to be lost.
2584          */
2585         switch (amdgpu_asic_reset_method(adev)) {
2586         case AMD_RESET_METHOD_BACO:
2587         case AMD_RESET_METHOD_MODE1:
2588                 return true;
2589         default:
2590                 return false;
2591         }
2592 }
2593
2594 /**
2595  * amdgpu_device_set_cg_state - set clockgating for amdgpu device
2596  *
2597  * @adev: amdgpu_device pointer
2598  * @state: clockgating state (gate or ungate)
2599  *
2600  * The list of all the hardware IPs that make up the asic is walked and the
2601  * set_clockgating_state callbacks are run.
2602  * Late initialization pass enabling clockgating for hardware IPs.
2603  * Fini or suspend, pass disabling clockgating for hardware IPs.
2604  * Returns 0 on success, negative error code on failure.
2605  */
2606
2607 int amdgpu_device_set_cg_state(struct amdgpu_device *adev,
2608                                enum amd_clockgating_state state)
2609 {
2610         int i, j, r;
2611
2612         if (amdgpu_emu_mode == 1)
2613                 return 0;
2614
2615         for (j = 0; j < adev->num_ip_blocks; j++) {
2616                 i = state == AMD_CG_STATE_GATE ? j : adev->num_ip_blocks - j - 1;
2617                 if (!adev->ip_blocks[i].status.late_initialized)
2618                         continue;
2619                 /* skip CG for GFX, SDMA on S0ix */
2620                 if (adev->in_s0ix &&
2621                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX ||
2622                      adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SDMA))
2623                         continue;
2624                 /* skip CG for VCE/UVD, it's handled specially */
2625                 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
2626                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&
2627                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&
2628                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG &&
2629                     adev->ip_blocks[i].version->funcs->set_clockgating_state) {
2630                         /* enable clockgating to save power */
2631                         r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
2632                                                                                      state);
2633                         if (r) {
2634                                 DRM_ERROR("set_clockgating_state(gate) of IP block <%s> failed %d\n",
2635                                           adev->ip_blocks[i].version->funcs->name, r);
2636                                 return r;
2637                         }
2638                 }
2639         }
2640
2641         return 0;
2642 }
2643
2644 int amdgpu_device_set_pg_state(struct amdgpu_device *adev,
2645                                enum amd_powergating_state state)
2646 {
2647         int i, j, r;
2648
2649         if (amdgpu_emu_mode == 1)
2650                 return 0;
2651
2652         for (j = 0; j < adev->num_ip_blocks; j++) {
2653                 i = state == AMD_PG_STATE_GATE ? j : adev->num_ip_blocks - j - 1;
2654                 if (!adev->ip_blocks[i].status.late_initialized)
2655                         continue;
2656                 /* skip PG for GFX, SDMA on S0ix */
2657                 if (adev->in_s0ix &&
2658                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX ||
2659                      adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SDMA))
2660                         continue;
2661                 /* skip CG for VCE/UVD, it's handled specially */
2662                 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
2663                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&
2664                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&
2665                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG &&
2666                     adev->ip_blocks[i].version->funcs->set_powergating_state) {
2667                         /* enable powergating to save power */
2668                         r = adev->ip_blocks[i].version->funcs->set_powergating_state((void *)adev,
2669                                                                                         state);
2670                         if (r) {
2671                                 DRM_ERROR("set_powergating_state(gate) of IP block <%s> failed %d\n",
2672                                           adev->ip_blocks[i].version->funcs->name, r);
2673                                 return r;
2674                         }
2675                 }
2676         }
2677         return 0;
2678 }
2679
2680 static int amdgpu_device_enable_mgpu_fan_boost(void)
2681 {
2682         struct amdgpu_gpu_instance *gpu_ins;
2683         struct amdgpu_device *adev;
2684         int i, ret = 0;
2685
2686         mutex_lock(&mgpu_info.mutex);
2687
2688         /*
2689          * MGPU fan boost feature should be enabled
2690          * only when there are two or more dGPUs in
2691          * the system
2692          */
2693         if (mgpu_info.num_dgpu < 2)
2694                 goto out;
2695
2696         for (i = 0; i < mgpu_info.num_dgpu; i++) {
2697                 gpu_ins = &(mgpu_info.gpu_ins[i]);
2698                 adev = gpu_ins->adev;
2699                 if (!(adev->flags & AMD_IS_APU) &&
2700                     !gpu_ins->mgpu_fan_enabled) {
2701                         ret = amdgpu_dpm_enable_mgpu_fan_boost(adev);
2702                         if (ret)
2703                                 break;
2704
2705                         gpu_ins->mgpu_fan_enabled = 1;
2706                 }
2707         }
2708
2709 out:
2710         mutex_unlock(&mgpu_info.mutex);
2711
2712         return ret;
2713 }
2714
2715 /**
2716  * amdgpu_device_ip_late_init - run late init for hardware IPs
2717  *
2718  * @adev: amdgpu_device pointer
2719  *
2720  * Late initialization pass for hardware IPs.  The list of all the hardware
2721  * IPs that make up the asic is walked and the late_init callbacks are run.
2722  * late_init covers any special initialization that an IP requires
2723  * after all of the have been initialized or something that needs to happen
2724  * late in the init process.
2725  * Returns 0 on success, negative error code on failure.
2726  */
2727 static int amdgpu_device_ip_late_init(struct amdgpu_device *adev)
2728 {
2729         struct amdgpu_gpu_instance *gpu_instance;
2730         int i = 0, r;
2731
2732         for (i = 0; i < adev->num_ip_blocks; i++) {
2733                 if (!adev->ip_blocks[i].status.hw)
2734                         continue;
2735                 if (adev->ip_blocks[i].version->funcs->late_init) {
2736                         r = adev->ip_blocks[i].version->funcs->late_init((void *)adev);
2737                         if (r) {
2738                                 DRM_ERROR("late_init of IP block <%s> failed %d\n",
2739                                           adev->ip_blocks[i].version->funcs->name, r);
2740                                 return r;
2741                         }
2742                 }
2743                 adev->ip_blocks[i].status.late_initialized = true;
2744         }
2745
2746         r = amdgpu_ras_late_init(adev);
2747         if (r) {
2748                 DRM_ERROR("amdgpu_ras_late_init failed %d", r);
2749                 return r;
2750         }
2751
2752         amdgpu_ras_set_error_query_ready(adev, true);
2753
2754         amdgpu_device_set_cg_state(adev, AMD_CG_STATE_GATE);
2755         amdgpu_device_set_pg_state(adev, AMD_PG_STATE_GATE);
2756
2757         amdgpu_device_fill_reset_magic(adev);
2758
2759         r = amdgpu_device_enable_mgpu_fan_boost();
2760         if (r)
2761                 DRM_ERROR("enable mgpu fan boost failed (%d).\n", r);
2762
2763         /* For passthrough configuration on arcturus and aldebaran, enable special handling SBR */
2764         if (amdgpu_passthrough(adev) && ((adev->asic_type == CHIP_ARCTURUS && adev->gmc.xgmi.num_physical_nodes > 1)||
2765                                adev->asic_type == CHIP_ALDEBARAN ))
2766                 amdgpu_dpm_handle_passthrough_sbr(adev, true);
2767
2768         if (adev->gmc.xgmi.num_physical_nodes > 1) {
2769                 mutex_lock(&mgpu_info.mutex);
2770
2771                 /*
2772                  * Reset device p-state to low as this was booted with high.
2773                  *
2774                  * This should be performed only after all devices from the same
2775                  * hive get initialized.
2776                  *
2777                  * However, it's unknown how many device in the hive in advance.
2778                  * As this is counted one by one during devices initializations.
2779                  *
2780                  * So, we wait for all XGMI interlinked devices initialized.
2781                  * This may bring some delays as those devices may come from
2782                  * different hives. But that should be OK.
2783                  */
2784                 if (mgpu_info.num_dgpu == adev->gmc.xgmi.num_physical_nodes) {
2785                         for (i = 0; i < mgpu_info.num_gpu; i++) {
2786                                 gpu_instance = &(mgpu_info.gpu_ins[i]);
2787                                 if (gpu_instance->adev->flags & AMD_IS_APU)
2788                                         continue;
2789
2790                                 r = amdgpu_xgmi_set_pstate(gpu_instance->adev,
2791                                                 AMDGPU_XGMI_PSTATE_MIN);
2792                                 if (r) {
2793                                         DRM_ERROR("pstate setting failed (%d).\n", r);
2794                                         break;
2795                                 }
2796                         }
2797                 }
2798
2799                 mutex_unlock(&mgpu_info.mutex);
2800         }
2801
2802         return 0;
2803 }
2804
2805 /**
2806  * amdgpu_device_smu_fini_early - smu hw_fini wrapper
2807  *
2808  * @adev: amdgpu_device pointer
2809  *
2810  * For ASICs need to disable SMC first
2811  */
2812 static void amdgpu_device_smu_fini_early(struct amdgpu_device *adev)
2813 {
2814         int i, r;
2815
2816         if (adev->ip_versions[GC_HWIP][0] > IP_VERSION(9, 0, 0))
2817                 return;
2818
2819         for (i = 0; i < adev->num_ip_blocks; i++) {
2820                 if (!adev->ip_blocks[i].status.hw)
2821                         continue;
2822                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
2823                         r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
2824                         /* XXX handle errors */
2825                         if (r) {
2826                                 DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
2827                                           adev->ip_blocks[i].version->funcs->name, r);
2828                         }
2829                         adev->ip_blocks[i].status.hw = false;
2830                         break;
2831                 }
2832         }
2833 }
2834
2835 static int amdgpu_device_ip_fini_early(struct amdgpu_device *adev)
2836 {
2837         int i, r;
2838
2839         for (i = 0; i < adev->num_ip_blocks; i++) {
2840                 if (!adev->ip_blocks[i].version->funcs->early_fini)
2841                         continue;
2842
2843                 r = adev->ip_blocks[i].version->funcs->early_fini((void *)adev);
2844                 if (r) {
2845                         DRM_DEBUG("early_fini of IP block <%s> failed %d\n",
2846                                   adev->ip_blocks[i].version->funcs->name, r);
2847                 }
2848         }
2849
2850         amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE);
2851         amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE);
2852
2853         amdgpu_amdkfd_suspend(adev, false);
2854
2855         /* Workaroud for ASICs need to disable SMC first */
2856         amdgpu_device_smu_fini_early(adev);
2857
2858         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2859                 if (!adev->ip_blocks[i].status.hw)
2860                         continue;
2861
2862                 r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
2863                 /* XXX handle errors */
2864                 if (r) {
2865                         DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
2866                                   adev->ip_blocks[i].version->funcs->name, r);
2867                 }
2868
2869                 adev->ip_blocks[i].status.hw = false;
2870         }
2871
2872         if (amdgpu_sriov_vf(adev)) {
2873                 if (amdgpu_virt_release_full_gpu(adev, false))
2874                         DRM_ERROR("failed to release exclusive mode on fini\n");
2875         }
2876
2877         return 0;
2878 }
2879
2880 /**
2881  * amdgpu_device_ip_fini - run fini for hardware IPs
2882  *
2883  * @adev: amdgpu_device pointer
2884  *
2885  * Main teardown pass for hardware IPs.  The list of all the hardware
2886  * IPs that make up the asic is walked and the hw_fini and sw_fini callbacks
2887  * are run.  hw_fini tears down the hardware associated with each IP
2888  * and sw_fini tears down any software state associated with each IP.
2889  * Returns 0 on success, negative error code on failure.
2890  */
2891 static int amdgpu_device_ip_fini(struct amdgpu_device *adev)
2892 {
2893         int i, r;
2894
2895         if (amdgpu_sriov_vf(adev) && adev->virt.ras_init_done)
2896                 amdgpu_virt_release_ras_err_handler_data(adev);
2897
2898         if (adev->gmc.xgmi.num_physical_nodes > 1)
2899                 amdgpu_xgmi_remove_device(adev);
2900
2901         amdgpu_amdkfd_device_fini_sw(adev);
2902
2903         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2904                 if (!adev->ip_blocks[i].status.sw)
2905                         continue;
2906
2907                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
2908                         amdgpu_ucode_free_bo(adev);
2909                         amdgpu_free_static_csa(&adev->virt.csa_obj);
2910                         amdgpu_device_wb_fini(adev);
2911                         amdgpu_device_mem_scratch_fini(adev);
2912                         amdgpu_ib_pool_fini(adev);
2913                 }
2914
2915                 r = adev->ip_blocks[i].version->funcs->sw_fini((void *)adev);
2916                 /* XXX handle errors */
2917                 if (r) {
2918                         DRM_DEBUG("sw_fini of IP block <%s> failed %d\n",
2919                                   adev->ip_blocks[i].version->funcs->name, r);
2920                 }
2921                 adev->ip_blocks[i].status.sw = false;
2922                 adev->ip_blocks[i].status.valid = false;
2923         }
2924
2925         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2926                 if (!adev->ip_blocks[i].status.late_initialized)
2927                         continue;
2928                 if (adev->ip_blocks[i].version->funcs->late_fini)
2929                         adev->ip_blocks[i].version->funcs->late_fini((void *)adev);
2930                 adev->ip_blocks[i].status.late_initialized = false;
2931         }
2932
2933         amdgpu_ras_fini(adev);
2934
2935         return 0;
2936 }
2937
2938 /**
2939  * amdgpu_device_delayed_init_work_handler - work handler for IB tests
2940  *
2941  * @work: work_struct.
2942  */
2943 static void amdgpu_device_delayed_init_work_handler(struct work_struct *work)
2944 {
2945         struct amdgpu_device *adev =
2946                 container_of(work, struct amdgpu_device, delayed_init_work.work);
2947         int r;
2948
2949         r = amdgpu_ib_ring_tests(adev);
2950         if (r)
2951                 DRM_ERROR("ib ring test failed (%d).\n", r);
2952 }
2953
2954 static void amdgpu_device_delay_enable_gfx_off(struct work_struct *work)
2955 {
2956         struct amdgpu_device *adev =
2957                 container_of(work, struct amdgpu_device, gfx.gfx_off_delay_work.work);
2958
2959         WARN_ON_ONCE(adev->gfx.gfx_off_state);
2960         WARN_ON_ONCE(adev->gfx.gfx_off_req_count);
2961
2962         if (!amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, true))
2963                 adev->gfx.gfx_off_state = true;
2964 }
2965
2966 /**
2967  * amdgpu_device_ip_suspend_phase1 - run suspend for hardware IPs (phase 1)
2968  *
2969  * @adev: amdgpu_device pointer
2970  *
2971  * Main suspend function for hardware IPs.  The list of all the hardware
2972  * IPs that make up the asic is walked, clockgating is disabled and the
2973  * suspend callbacks are run.  suspend puts the hardware and software state
2974  * in each IP into a state suitable for suspend.
2975  * Returns 0 on success, negative error code on failure.
2976  */
2977 static int amdgpu_device_ip_suspend_phase1(struct amdgpu_device *adev)
2978 {
2979         int i, r;
2980
2981         amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE);
2982         amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE);
2983
2984         /*
2985          * Per PMFW team's suggestion, driver needs to handle gfxoff
2986          * and df cstate features disablement for gpu reset(e.g. Mode1Reset)
2987          * scenario. Add the missing df cstate disablement here.
2988          */
2989         if (amdgpu_dpm_set_df_cstate(adev, DF_CSTATE_DISALLOW))
2990                 dev_warn(adev->dev, "Failed to disallow df cstate");
2991
2992         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2993                 if (!adev->ip_blocks[i].status.valid)
2994                         continue;
2995
2996                 /* displays are handled separately */
2997                 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_DCE)
2998                         continue;
2999
3000                 /* XXX handle errors */
3001                 r = adev->ip_blocks[i].version->funcs->suspend(adev);
3002                 /* XXX handle errors */
3003                 if (r) {
3004                         DRM_ERROR("suspend of IP block <%s> failed %d\n",
3005                                   adev->ip_blocks[i].version->funcs->name, r);
3006                         return r;
3007                 }
3008
3009                 adev->ip_blocks[i].status.hw = false;
3010         }
3011
3012         return 0;
3013 }
3014
3015 /**
3016  * amdgpu_device_ip_suspend_phase2 - run suspend for hardware IPs (phase 2)
3017  *
3018  * @adev: amdgpu_device pointer
3019  *
3020  * Main suspend function for hardware IPs.  The list of all the hardware
3021  * IPs that make up the asic is walked, clockgating is disabled and the
3022  * suspend callbacks are run.  suspend puts the hardware and software state
3023  * in each IP into a state suitable for suspend.
3024  * Returns 0 on success, negative error code on failure.
3025  */
3026 static int amdgpu_device_ip_suspend_phase2(struct amdgpu_device *adev)
3027 {
3028         int i, r;
3029
3030         if (adev->in_s0ix)
3031                 amdgpu_dpm_gfx_state_change(adev, sGpuChangeState_D3Entry);
3032
3033         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
3034                 if (!adev->ip_blocks[i].status.valid)
3035                         continue;
3036                 /* displays are handled in phase1 */
3037                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE)
3038                         continue;
3039                 /* PSP lost connection when err_event_athub occurs */
3040                 if (amdgpu_ras_intr_triggered() &&
3041                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) {
3042                         adev->ip_blocks[i].status.hw = false;
3043                         continue;
3044                 }
3045
3046                 /* skip unnecessary suspend if we do not initialize them yet */
3047                 if (adev->gmc.xgmi.pending_reset &&
3048                     !(adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3049                       adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC ||
3050                       adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3051                       adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH)) {
3052                         adev->ip_blocks[i].status.hw = false;
3053                         continue;
3054                 }
3055
3056                 /* skip suspend of gfx/mes and psp for S0ix
3057                  * gfx is in gfxoff state, so on resume it will exit gfxoff just
3058                  * like at runtime. PSP is also part of the always on hardware
3059                  * so no need to suspend it.
3060                  */
3061                 if (adev->in_s0ix &&
3062                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP ||
3063                      adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX ||
3064                      adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_MES))
3065                         continue;
3066
3067                 /* SDMA 5.x+ is part of GFX power domain so it's covered by GFXOFF */
3068                 if (adev->in_s0ix &&
3069                     (adev->ip_versions[SDMA0_HWIP][0] >= IP_VERSION(5, 0, 0)) &&
3070                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SDMA))
3071                         continue;
3072
3073                 /* Once swPSP provides the IMU, RLC FW binaries to TOS during cold-boot.
3074                  * These are in TMR, hence are expected to be reused by PSP-TOS to reload
3075                  * from this location and RLC Autoload automatically also gets loaded
3076                  * from here based on PMFW -> PSP message during re-init sequence.
3077                  * Therefore, the psp suspend & resume should be skipped to avoid destroy
3078                  * the TMR and reload FWs again for IMU enabled APU ASICs.
3079                  */
3080                 if (amdgpu_in_reset(adev) &&
3081                     (adev->flags & AMD_IS_APU) && adev->gfx.imu.funcs &&
3082                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)
3083                         continue;
3084
3085                 /* XXX handle errors */
3086                 r = adev->ip_blocks[i].version->funcs->suspend(adev);
3087                 /* XXX handle errors */
3088                 if (r) {
3089                         DRM_ERROR("suspend of IP block <%s> failed %d\n",
3090                                   adev->ip_blocks[i].version->funcs->name, r);
3091                 }
3092                 adev->ip_blocks[i].status.hw = false;
3093                 /* handle putting the SMC in the appropriate state */
3094                 if(!amdgpu_sriov_vf(adev)){
3095                         if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
3096                                 r = amdgpu_dpm_set_mp1_state(adev, adev->mp1_state);
3097                                 if (r) {
3098                                         DRM_ERROR("SMC failed to set mp1 state %d, %d\n",
3099                                                         adev->mp1_state, r);
3100                                         return r;
3101                                 }
3102                         }
3103                 }
3104         }
3105
3106         return 0;
3107 }
3108
3109 /**
3110  * amdgpu_device_ip_suspend - run suspend for hardware IPs
3111  *
3112  * @adev: amdgpu_device pointer
3113  *
3114  * Main suspend function for hardware IPs.  The list of all the hardware
3115  * IPs that make up the asic is walked, clockgating is disabled and the
3116  * suspend callbacks are run.  suspend puts the hardware and software state
3117  * in each IP into a state suitable for suspend.
3118  * Returns 0 on success, negative error code on failure.
3119  */
3120 int amdgpu_device_ip_suspend(struct amdgpu_device *adev)
3121 {
3122         int r;
3123
3124         if (amdgpu_sriov_vf(adev)) {
3125                 amdgpu_virt_fini_data_exchange(adev);
3126                 amdgpu_virt_request_full_gpu(adev, false);
3127         }
3128
3129         r = amdgpu_device_ip_suspend_phase1(adev);
3130         if (r)
3131                 return r;
3132         r = amdgpu_device_ip_suspend_phase2(adev);
3133
3134         if (amdgpu_sriov_vf(adev))
3135                 amdgpu_virt_release_full_gpu(adev, false);
3136
3137         return r;
3138 }
3139
3140 static int amdgpu_device_ip_reinit_early_sriov(struct amdgpu_device *adev)
3141 {
3142         int i, r;
3143
3144         static enum amd_ip_block_type ip_order[] = {
3145                 AMD_IP_BLOCK_TYPE_COMMON,
3146                 AMD_IP_BLOCK_TYPE_GMC,
3147                 AMD_IP_BLOCK_TYPE_PSP,
3148                 AMD_IP_BLOCK_TYPE_IH,
3149         };
3150
3151         for (i = 0; i < adev->num_ip_blocks; i++) {
3152                 int j;
3153                 struct amdgpu_ip_block *block;
3154
3155                 block = &adev->ip_blocks[i];
3156                 block->status.hw = false;
3157
3158                 for (j = 0; j < ARRAY_SIZE(ip_order); j++) {
3159
3160                         if (block->version->type != ip_order[j] ||
3161                                 !block->status.valid)
3162                                 continue;
3163
3164                         r = block->version->funcs->hw_init(adev);
3165                         DRM_INFO("RE-INIT-early: %s %s\n", block->version->funcs->name, r?"failed":"succeeded");
3166                         if (r)
3167                                 return r;
3168                         block->status.hw = true;
3169                 }
3170         }
3171
3172         return 0;
3173 }
3174
3175 static int amdgpu_device_ip_reinit_late_sriov(struct amdgpu_device *adev)
3176 {
3177         int i, r;
3178
3179         static enum amd_ip_block_type ip_order[] = {
3180                 AMD_IP_BLOCK_TYPE_SMC,
3181                 AMD_IP_BLOCK_TYPE_DCE,
3182                 AMD_IP_BLOCK_TYPE_GFX,
3183                 AMD_IP_BLOCK_TYPE_SDMA,
3184                 AMD_IP_BLOCK_TYPE_MES,
3185                 AMD_IP_BLOCK_TYPE_UVD,
3186                 AMD_IP_BLOCK_TYPE_VCE,
3187                 AMD_IP_BLOCK_TYPE_VCN,
3188                 AMD_IP_BLOCK_TYPE_JPEG
3189         };
3190
3191         for (i = 0; i < ARRAY_SIZE(ip_order); i++) {
3192                 int j;
3193                 struct amdgpu_ip_block *block;
3194
3195                 for (j = 0; j < adev->num_ip_blocks; j++) {
3196                         block = &adev->ip_blocks[j];
3197
3198                         if (block->version->type != ip_order[i] ||
3199                                 !block->status.valid ||
3200                                 block->status.hw)
3201                                 continue;
3202
3203                         if (block->version->type == AMD_IP_BLOCK_TYPE_SMC)
3204                                 r = block->version->funcs->resume(adev);
3205                         else
3206                                 r = block->version->funcs->hw_init(adev);
3207
3208                         DRM_INFO("RE-INIT-late: %s %s\n", block->version->funcs->name, r?"failed":"succeeded");
3209                         if (r)
3210                                 return r;
3211                         block->status.hw = true;
3212                 }
3213         }
3214
3215         return 0;
3216 }
3217
3218 /**
3219  * amdgpu_device_ip_resume_phase1 - run resume for hardware IPs
3220  *
3221  * @adev: amdgpu_device pointer
3222  *
3223  * First resume function for hardware IPs.  The list of all the hardware
3224  * IPs that make up the asic is walked and the resume callbacks are run for
3225  * COMMON, GMC, and IH.  resume puts the hardware into a functional state
3226  * after a suspend and updates the software state as necessary.  This
3227  * function is also used for restoring the GPU after a GPU reset.
3228  * Returns 0 on success, negative error code on failure.
3229  */
3230 static int amdgpu_device_ip_resume_phase1(struct amdgpu_device *adev)
3231 {
3232         int i, r;
3233
3234         for (i = 0; i < adev->num_ip_blocks; i++) {
3235                 if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw)
3236                         continue;
3237                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3238                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3239                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||
3240                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP && amdgpu_sriov_vf(adev))) {
3241
3242                         r = adev->ip_blocks[i].version->funcs->resume(adev);
3243                         if (r) {
3244                                 DRM_ERROR("resume of IP block <%s> failed %d\n",
3245                                           adev->ip_blocks[i].version->funcs->name, r);
3246                                 return r;
3247                         }
3248                         adev->ip_blocks[i].status.hw = true;
3249                 }
3250         }
3251
3252         return 0;
3253 }
3254
3255 /**
3256  * amdgpu_device_ip_resume_phase2 - run resume for hardware IPs
3257  *
3258  * @adev: amdgpu_device pointer
3259  *
3260  * First resume function for hardware IPs.  The list of all the hardware
3261  * IPs that make up the asic is walked and the resume callbacks are run for
3262  * all blocks except COMMON, GMC, and IH.  resume puts the hardware into a
3263  * functional state after a suspend and updates the software state as
3264  * necessary.  This function is also used for restoring the GPU after a GPU
3265  * reset.
3266  * Returns 0 on success, negative error code on failure.
3267  */
3268 static int amdgpu_device_ip_resume_phase2(struct amdgpu_device *adev)
3269 {
3270         int i, r;
3271
3272         for (i = 0; i < adev->num_ip_blocks; i++) {
3273                 if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw)
3274                         continue;
3275                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3276                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3277                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||
3278                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)
3279                         continue;
3280                 r = adev->ip_blocks[i].version->funcs->resume(adev);
3281                 if (r) {
3282                         DRM_ERROR("resume of IP block <%s> failed %d\n",
3283                                   adev->ip_blocks[i].version->funcs->name, r);
3284                         return r;
3285                 }
3286                 adev->ip_blocks[i].status.hw = true;
3287         }
3288
3289         return 0;
3290 }
3291
3292 /**
3293  * amdgpu_device_ip_resume - run resume for hardware IPs
3294  *
3295  * @adev: amdgpu_device pointer
3296  *
3297  * Main resume function for hardware IPs.  The hardware IPs
3298  * are split into two resume functions because they are
3299  * are also used in in recovering from a GPU reset and some additional
3300  * steps need to be take between them.  In this case (S3/S4) they are
3301  * run sequentially.
3302  * Returns 0 on success, negative error code on failure.
3303  */
3304 static int amdgpu_device_ip_resume(struct amdgpu_device *adev)
3305 {
3306         int r;
3307
3308         if (!adev->in_s0ix) {
3309                 r = amdgpu_amdkfd_resume_iommu(adev);
3310                 if (r)
3311                         return r;
3312         }
3313
3314         r = amdgpu_device_ip_resume_phase1(adev);
3315         if (r)
3316                 return r;
3317
3318         r = amdgpu_device_fw_loading(adev);
3319         if (r)
3320                 return r;
3321
3322         r = amdgpu_device_ip_resume_phase2(adev);
3323
3324         return r;
3325 }
3326
3327 /**
3328  * amdgpu_device_detect_sriov_bios - determine if the board supports SR-IOV
3329  *
3330  * @adev: amdgpu_device pointer
3331  *
3332  * Query the VBIOS data tables to determine if the board supports SR-IOV.
3333  */
3334 static void amdgpu_device_detect_sriov_bios(struct amdgpu_device *adev)
3335 {
3336         if (amdgpu_sriov_vf(adev)) {
3337                 if (adev->is_atom_fw) {
3338                         if (amdgpu_atomfirmware_gpu_virtualization_supported(adev))
3339                                 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
3340                 } else {
3341                         if (amdgpu_atombios_has_gpu_virtualization_table(adev))
3342                                 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
3343                 }
3344
3345                 if (!(adev->virt.caps & AMDGPU_SRIOV_CAPS_SRIOV_VBIOS))
3346                         amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_NO_VBIOS, 0, 0);
3347         }
3348 }
3349
3350 /**
3351  * amdgpu_device_asic_has_dc_support - determine if DC supports the asic
3352  *
3353  * @asic_type: AMD asic type
3354  *
3355  * Check if there is DC (new modesetting infrastructre) support for an asic.
3356  * returns true if DC has support, false if not.
3357  */
3358 bool amdgpu_device_asic_has_dc_support(enum amd_asic_type asic_type)
3359 {
3360         switch (asic_type) {
3361 #ifdef CONFIG_DRM_AMDGPU_SI
3362         case CHIP_HAINAN:
3363 #endif
3364         case CHIP_TOPAZ:
3365                 /* chips with no display hardware */
3366                 return false;
3367 #if defined(CONFIG_DRM_AMD_DC)
3368         case CHIP_TAHITI:
3369         case CHIP_PITCAIRN:
3370         case CHIP_VERDE:
3371         case CHIP_OLAND:
3372                 /*
3373                  * We have systems in the wild with these ASICs that require
3374                  * LVDS and VGA support which is not supported with DC.
3375                  *
3376                  * Fallback to the non-DC driver here by default so as not to
3377                  * cause regressions.
3378                  */
3379 #if defined(CONFIG_DRM_AMD_DC_SI)
3380                 return amdgpu_dc > 0;
3381 #else
3382                 return false;
3383 #endif
3384         case CHIP_BONAIRE:
3385         case CHIP_KAVERI:
3386         case CHIP_KABINI:
3387         case CHIP_MULLINS:
3388                 /*
3389                  * We have systems in the wild with these ASICs that require
3390                  * VGA support which is not supported with DC.
3391                  *
3392                  * Fallback to the non-DC driver here by default so as not to
3393                  * cause regressions.
3394                  */
3395                 return amdgpu_dc > 0;
3396         default:
3397                 return amdgpu_dc != 0;
3398 #else
3399         default:
3400                 if (amdgpu_dc > 0)
3401                         DRM_INFO_ONCE("Display Core has been requested via kernel parameter "
3402                                          "but isn't supported by ASIC, ignoring\n");
3403                 return false;
3404 #endif
3405         }
3406 }
3407
3408 /**
3409  * amdgpu_device_has_dc_support - check if dc is supported
3410  *
3411  * @adev: amdgpu_device pointer
3412  *
3413  * Returns true for supported, false for not supported
3414  */
3415 bool amdgpu_device_has_dc_support(struct amdgpu_device *adev)
3416 {
3417         if (adev->enable_virtual_display ||
3418             (adev->harvest_ip_mask & AMD_HARVEST_IP_DMU_MASK))
3419                 return false;
3420
3421         return amdgpu_device_asic_has_dc_support(adev->asic_type);
3422 }
3423
3424 static void amdgpu_device_xgmi_reset_func(struct work_struct *__work)
3425 {
3426         struct amdgpu_device *adev =
3427                 container_of(__work, struct amdgpu_device, xgmi_reset_work);
3428         struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
3429
3430         /* It's a bug to not have a hive within this function */
3431         if (WARN_ON(!hive))
3432                 return;
3433
3434         /*
3435          * Use task barrier to synchronize all xgmi reset works across the
3436          * hive. task_barrier_enter and task_barrier_exit will block
3437          * until all the threads running the xgmi reset works reach
3438          * those points. task_barrier_full will do both blocks.
3439          */
3440         if (amdgpu_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) {
3441
3442                 task_barrier_enter(&hive->tb);
3443                 adev->asic_reset_res = amdgpu_device_baco_enter(adev_to_drm(adev));
3444
3445                 if (adev->asic_reset_res)
3446                         goto fail;
3447
3448                 task_barrier_exit(&hive->tb);
3449                 adev->asic_reset_res = amdgpu_device_baco_exit(adev_to_drm(adev));
3450
3451                 if (adev->asic_reset_res)
3452                         goto fail;
3453
3454                 if (adev->mmhub.ras && adev->mmhub.ras->ras_block.hw_ops &&
3455                     adev->mmhub.ras->ras_block.hw_ops->reset_ras_error_count)
3456                         adev->mmhub.ras->ras_block.hw_ops->reset_ras_error_count(adev);
3457         } else {
3458
3459                 task_barrier_full(&hive->tb);
3460                 adev->asic_reset_res =  amdgpu_asic_reset(adev);
3461         }
3462
3463 fail:
3464         if (adev->asic_reset_res)
3465                 DRM_WARN("ASIC reset failed with error, %d for drm dev, %s",
3466                          adev->asic_reset_res, adev_to_drm(adev)->unique);
3467         amdgpu_put_xgmi_hive(hive);
3468 }
3469
3470 static int amdgpu_device_get_job_timeout_settings(struct amdgpu_device *adev)
3471 {
3472         char *input = amdgpu_lockup_timeout;
3473         char *timeout_setting = NULL;
3474         int index = 0;
3475         long timeout;
3476         int ret = 0;
3477
3478         /*
3479          * By default timeout for non compute jobs is 10000
3480          * and 60000 for compute jobs.
3481          * In SR-IOV or passthrough mode, timeout for compute
3482          * jobs are 60000 by default.
3483          */
3484         adev->gfx_timeout = msecs_to_jiffies(10000);
3485         adev->sdma_timeout = adev->video_timeout = adev->gfx_timeout;
3486         if (amdgpu_sriov_vf(adev))
3487                 adev->compute_timeout = amdgpu_sriov_is_pp_one_vf(adev) ?
3488                                         msecs_to_jiffies(60000) : msecs_to_jiffies(10000);
3489         else
3490                 adev->compute_timeout =  msecs_to_jiffies(60000);
3491
3492         if (strnlen(input, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH)) {
3493                 while ((timeout_setting = strsep(&input, ",")) &&
3494                                 strnlen(timeout_setting, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH)) {
3495                         ret = kstrtol(timeout_setting, 0, &timeout);
3496                         if (ret)
3497                                 return ret;
3498
3499                         if (timeout == 0) {
3500                                 index++;
3501                                 continue;
3502                         } else if (timeout < 0) {
3503                                 timeout = MAX_SCHEDULE_TIMEOUT;
3504                                 dev_warn(adev->dev, "lockup timeout disabled");
3505                                 add_taint(TAINT_SOFTLOCKUP, LOCKDEP_STILL_OK);
3506                         } else {
3507                                 timeout = msecs_to_jiffies(timeout);
3508                         }
3509
3510                         switch (index++) {
3511                         case 0:
3512                                 adev->gfx_timeout = timeout;
3513                                 break;
3514                         case 1:
3515                                 adev->compute_timeout = timeout;
3516                                 break;
3517                         case 2:
3518                                 adev->sdma_timeout = timeout;
3519                                 break;
3520                         case 3:
3521                                 adev->video_timeout = timeout;
3522                                 break;
3523                         default:
3524                                 break;
3525                         }
3526                 }
3527                 /*
3528                  * There is only one value specified and
3529                  * it should apply to all non-compute jobs.
3530                  */
3531                 if (index == 1) {
3532                         adev->sdma_timeout = adev->video_timeout = adev->gfx_timeout;
3533                         if (amdgpu_sriov_vf(adev) || amdgpu_passthrough(adev))
3534                                 adev->compute_timeout = adev->gfx_timeout;
3535                 }
3536         }
3537
3538         return ret;
3539 }
3540
3541 /**
3542  * amdgpu_device_check_iommu_direct_map - check if RAM direct mapped to GPU
3543  *
3544  * @adev: amdgpu_device pointer
3545  *
3546  * RAM direct mapped to GPU if IOMMU is not enabled or is pass through mode
3547  */
3548 static void amdgpu_device_check_iommu_direct_map(struct amdgpu_device *adev)
3549 {
3550         struct iommu_domain *domain;
3551
3552         domain = iommu_get_domain_for_dev(adev->dev);
3553         if (!domain || domain->type == IOMMU_DOMAIN_IDENTITY)
3554                 adev->ram_is_direct_mapped = true;
3555 }
3556
3557 static const struct attribute *amdgpu_dev_attributes[] = {
3558         &dev_attr_product_name.attr,
3559         &dev_attr_product_number.attr,
3560         &dev_attr_serial_number.attr,
3561         &dev_attr_pcie_replay_count.attr,
3562         NULL
3563 };
3564
3565 /**
3566  * amdgpu_device_init - initialize the driver
3567  *
3568  * @adev: amdgpu_device pointer
3569  * @flags: driver flags
3570  *
3571  * Initializes the driver info and hw (all asics).
3572  * Returns 0 for success or an error on failure.
3573  * Called at driver startup.
3574  */
3575 int amdgpu_device_init(struct amdgpu_device *adev,
3576                        uint32_t flags)
3577 {
3578         struct drm_device *ddev = adev_to_drm(adev);
3579         struct pci_dev *pdev = adev->pdev;
3580         int r, i;
3581         bool px = false;
3582         u32 max_MBps;
3583
3584         adev->shutdown = false;
3585         adev->flags = flags;
3586
3587         if (amdgpu_force_asic_type >= 0 && amdgpu_force_asic_type < CHIP_LAST)
3588                 adev->asic_type = amdgpu_force_asic_type;
3589         else
3590                 adev->asic_type = flags & AMD_ASIC_MASK;
3591
3592         adev->usec_timeout = AMDGPU_MAX_USEC_TIMEOUT;
3593         if (amdgpu_emu_mode == 1)
3594                 adev->usec_timeout *= 10;
3595         adev->gmc.gart_size = 512 * 1024 * 1024;
3596         adev->accel_working = false;
3597         adev->num_rings = 0;
3598         RCU_INIT_POINTER(adev->gang_submit, dma_fence_get_stub());
3599         adev->mman.buffer_funcs = NULL;
3600         adev->mman.buffer_funcs_ring = NULL;
3601         adev->vm_manager.vm_pte_funcs = NULL;
3602         adev->vm_manager.vm_pte_num_scheds = 0;
3603         adev->gmc.gmc_funcs = NULL;
3604         adev->harvest_ip_mask = 0x0;
3605         adev->fence_context = dma_fence_context_alloc(AMDGPU_MAX_RINGS);
3606         bitmap_zero(adev->gfx.pipe_reserve_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
3607
3608         adev->smc_rreg = &amdgpu_invalid_rreg;
3609         adev->smc_wreg = &amdgpu_invalid_wreg;
3610         adev->pcie_rreg = &amdgpu_invalid_rreg;
3611         adev->pcie_wreg = &amdgpu_invalid_wreg;
3612         adev->pciep_rreg = &amdgpu_invalid_rreg;
3613         adev->pciep_wreg = &amdgpu_invalid_wreg;
3614         adev->pcie_rreg64 = &amdgpu_invalid_rreg64;
3615         adev->pcie_wreg64 = &amdgpu_invalid_wreg64;
3616         adev->uvd_ctx_rreg = &amdgpu_invalid_rreg;
3617         adev->uvd_ctx_wreg = &amdgpu_invalid_wreg;
3618         adev->didt_rreg = &amdgpu_invalid_rreg;
3619         adev->didt_wreg = &amdgpu_invalid_wreg;
3620         adev->gc_cac_rreg = &amdgpu_invalid_rreg;
3621         adev->gc_cac_wreg = &amdgpu_invalid_wreg;
3622         adev->audio_endpt_rreg = &amdgpu_block_invalid_rreg;
3623         adev->audio_endpt_wreg = &amdgpu_block_invalid_wreg;
3624
3625         DRM_INFO("initializing kernel modesetting (%s 0x%04X:0x%04X 0x%04X:0x%04X 0x%02X).\n",
3626                  amdgpu_asic_name[adev->asic_type], pdev->vendor, pdev->device,
3627                  pdev->subsystem_vendor, pdev->subsystem_device, pdev->revision);
3628
3629         /* mutex initialization are all done here so we
3630          * can recall function without having locking issues */
3631         mutex_init(&adev->firmware.mutex);
3632         mutex_init(&adev->pm.mutex);
3633         mutex_init(&adev->gfx.gpu_clock_mutex);
3634         mutex_init(&adev->srbm_mutex);
3635         mutex_init(&adev->gfx.pipe_reserve_mutex);
3636         mutex_init(&adev->gfx.gfx_off_mutex);
3637         mutex_init(&adev->grbm_idx_mutex);
3638         mutex_init(&adev->mn_lock);
3639         mutex_init(&adev->virt.vf_errors.lock);
3640         hash_init(adev->mn_hash);
3641         mutex_init(&adev->psp.mutex);
3642         mutex_init(&adev->notifier_lock);
3643         mutex_init(&adev->pm.stable_pstate_ctx_lock);
3644         mutex_init(&adev->benchmark_mutex);
3645
3646         amdgpu_device_init_apu_flags(adev);
3647
3648         r = amdgpu_device_check_arguments(adev);
3649         if (r)
3650                 return r;
3651
3652         spin_lock_init(&adev->mmio_idx_lock);
3653         spin_lock_init(&adev->smc_idx_lock);
3654         spin_lock_init(&adev->pcie_idx_lock);
3655         spin_lock_init(&adev->uvd_ctx_idx_lock);
3656         spin_lock_init(&adev->didt_idx_lock);
3657         spin_lock_init(&adev->gc_cac_idx_lock);
3658         spin_lock_init(&adev->se_cac_idx_lock);
3659         spin_lock_init(&adev->audio_endpt_idx_lock);
3660         spin_lock_init(&adev->mm_stats.lock);
3661
3662         INIT_LIST_HEAD(&adev->shadow_list);
3663         mutex_init(&adev->shadow_list_lock);
3664
3665         INIT_LIST_HEAD(&adev->reset_list);
3666
3667         INIT_LIST_HEAD(&adev->ras_list);
3668
3669         INIT_DELAYED_WORK(&adev->delayed_init_work,
3670                           amdgpu_device_delayed_init_work_handler);
3671         INIT_DELAYED_WORK(&adev->gfx.gfx_off_delay_work,
3672                           amdgpu_device_delay_enable_gfx_off);
3673
3674         INIT_WORK(&adev->xgmi_reset_work, amdgpu_device_xgmi_reset_func);
3675
3676         adev->gfx.gfx_off_req_count = 1;
3677         adev->gfx.gfx_off_residency = 0;
3678         adev->gfx.gfx_off_entrycount = 0;
3679         adev->pm.ac_power = power_supply_is_system_supplied() > 0;
3680
3681         atomic_set(&adev->throttling_logging_enabled, 1);
3682         /*
3683          * If throttling continues, logging will be performed every minute
3684          * to avoid log flooding. "-1" is subtracted since the thermal
3685          * throttling interrupt comes every second. Thus, the total logging
3686          * interval is 59 seconds(retelimited printk interval) + 1(waiting
3687          * for throttling interrupt) = 60 seconds.
3688          */
3689         ratelimit_state_init(&adev->throttling_logging_rs, (60 - 1) * HZ, 1);
3690         ratelimit_set_flags(&adev->throttling_logging_rs, RATELIMIT_MSG_ON_RELEASE);
3691
3692         /* Registers mapping */
3693         /* TODO: block userspace mapping of io register */
3694         if (adev->asic_type >= CHIP_BONAIRE) {
3695                 adev->rmmio_base = pci_resource_start(adev->pdev, 5);
3696                 adev->rmmio_size = pci_resource_len(adev->pdev, 5);
3697         } else {
3698                 adev->rmmio_base = pci_resource_start(adev->pdev, 2);
3699                 adev->rmmio_size = pci_resource_len(adev->pdev, 2);
3700         }
3701
3702         for (i = 0; i < AMD_IP_BLOCK_TYPE_NUM; i++)
3703                 atomic_set(&adev->pm.pwr_state[i], POWER_STATE_UNKNOWN);
3704
3705         adev->rmmio = ioremap(adev->rmmio_base, adev->rmmio_size);
3706         if (adev->rmmio == NULL) {
3707                 return -ENOMEM;
3708         }
3709         DRM_INFO("register mmio base: 0x%08X\n", (uint32_t)adev->rmmio_base);
3710         DRM_INFO("register mmio size: %u\n", (unsigned)adev->rmmio_size);
3711
3712         amdgpu_device_get_pcie_info(adev);
3713
3714         if (amdgpu_mcbp)
3715                 DRM_INFO("MCBP is enabled\n");
3716
3717         /*
3718          * Reset domain needs to be present early, before XGMI hive discovered
3719          * (if any) and intitialized to use reset sem and in_gpu reset flag
3720          * early on during init and before calling to RREG32.
3721          */
3722         adev->reset_domain = amdgpu_reset_create_reset_domain(SINGLE_DEVICE, "amdgpu-reset-dev");
3723         if (!adev->reset_domain)
3724                 return -ENOMEM;
3725
3726         /* detect hw virtualization here */
3727         amdgpu_detect_virtualization(adev);
3728
3729         r = amdgpu_device_get_job_timeout_settings(adev);
3730         if (r) {
3731                 dev_err(adev->dev, "invalid lockup_timeout parameter syntax\n");
3732                 return r;
3733         }
3734
3735         /* early init functions */
3736         r = amdgpu_device_ip_early_init(adev);
3737         if (r)
3738                 return r;
3739
3740         /* Get rid of things like offb */
3741         r = drm_aperture_remove_conflicting_pci_framebuffers(adev->pdev, &amdgpu_kms_driver);
3742         if (r)
3743                 return r;
3744
3745         /* Enable TMZ based on IP_VERSION */
3746         amdgpu_gmc_tmz_set(adev);
3747
3748         amdgpu_gmc_noretry_set(adev);
3749         /* Need to get xgmi info early to decide the reset behavior*/
3750         if (adev->gmc.xgmi.supported) {
3751                 r = adev->gfxhub.funcs->get_xgmi_info(adev);
3752                 if (r)
3753                         return r;
3754         }
3755
3756         /* enable PCIE atomic ops */
3757         if (amdgpu_sriov_vf(adev))
3758                 adev->have_atomics_support = ((struct amd_sriov_msg_pf2vf_info *)
3759                         adev->virt.fw_reserve.p_pf2vf)->pcie_atomic_ops_support_flags ==
3760                         (PCI_EXP_DEVCAP2_ATOMIC_COMP32 | PCI_EXP_DEVCAP2_ATOMIC_COMP64);
3761         else
3762                 adev->have_atomics_support =
3763                         !pci_enable_atomic_ops_to_root(adev->pdev,
3764                                           PCI_EXP_DEVCAP2_ATOMIC_COMP32 |
3765                                           PCI_EXP_DEVCAP2_ATOMIC_COMP64);
3766         if (!adev->have_atomics_support)
3767                 dev_info(adev->dev, "PCIE atomic ops is not supported\n");
3768
3769         /* doorbell bar mapping and doorbell index init*/
3770         amdgpu_device_doorbell_init(adev);
3771
3772         if (amdgpu_emu_mode == 1) {
3773                 /* post the asic on emulation mode */
3774                 emu_soc_asic_init(adev);
3775                 goto fence_driver_init;
3776         }
3777
3778         amdgpu_reset_init(adev);
3779
3780         /* detect if we are with an SRIOV vbios */
3781         amdgpu_device_detect_sriov_bios(adev);
3782
3783         /* check if we need to reset the asic
3784          *  E.g., driver was not cleanly unloaded previously, etc.
3785          */
3786         if (!amdgpu_sriov_vf(adev) && amdgpu_asic_need_reset_on_init(adev)) {
3787                 if (adev->gmc.xgmi.num_physical_nodes) {
3788                         dev_info(adev->dev, "Pending hive reset.\n");
3789                         adev->gmc.xgmi.pending_reset = true;
3790                         /* Only need to init necessary block for SMU to handle the reset */
3791                         for (i = 0; i < adev->num_ip_blocks; i++) {
3792                                 if (!adev->ip_blocks[i].status.valid)
3793                                         continue;
3794                                 if (!(adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3795                                       adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3796                                       adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||
3797                                       adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC)) {
3798                                         DRM_DEBUG("IP %s disabled for hw_init.\n",
3799                                                 adev->ip_blocks[i].version->funcs->name);
3800                                         adev->ip_blocks[i].status.hw = true;
3801                                 }
3802                         }
3803                 } else {
3804                         r = amdgpu_asic_reset(adev);
3805                         if (r) {
3806                                 dev_err(adev->dev, "asic reset on init failed\n");
3807                                 goto failed;
3808                         }
3809                 }
3810         }
3811
3812         /* Post card if necessary */
3813         if (amdgpu_device_need_post(adev)) {
3814                 if (!adev->bios) {
3815                         dev_err(adev->dev, "no vBIOS found\n");
3816                         r = -EINVAL;
3817                         goto failed;
3818                 }
3819                 DRM_INFO("GPU posting now...\n");
3820                 r = amdgpu_device_asic_init(adev);
3821                 if (r) {
3822                         dev_err(adev->dev, "gpu post error!\n");
3823                         goto failed;
3824                 }
3825         }
3826
3827         if (adev->is_atom_fw) {
3828                 /* Initialize clocks */
3829                 r = amdgpu_atomfirmware_get_clock_info(adev);
3830                 if (r) {
3831                         dev_err(adev->dev, "amdgpu_atomfirmware_get_clock_info failed\n");
3832                         amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
3833                         goto failed;
3834                 }
3835         } else {
3836                 /* Initialize clocks */
3837                 r = amdgpu_atombios_get_clock_info(adev);
3838                 if (r) {
3839                         dev_err(adev->dev, "amdgpu_atombios_get_clock_info failed\n");
3840                         amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
3841                         goto failed;
3842                 }
3843                 /* init i2c buses */
3844                 if (!amdgpu_device_has_dc_support(adev))
3845                         amdgpu_atombios_i2c_init(adev);
3846         }
3847
3848 fence_driver_init:
3849         /* Fence driver */
3850         r = amdgpu_fence_driver_sw_init(adev);
3851         if (r) {
3852                 dev_err(adev->dev, "amdgpu_fence_driver_sw_init failed\n");
3853                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_FENCE_INIT_FAIL, 0, 0);
3854                 goto failed;
3855         }
3856
3857         /* init the mode config */
3858         drm_mode_config_init(adev_to_drm(adev));
3859
3860         r = amdgpu_device_ip_init(adev);
3861         if (r) {
3862                 /* failed in exclusive mode due to timeout */
3863                 if (amdgpu_sriov_vf(adev) &&
3864                     !amdgpu_sriov_runtime(adev) &&
3865                     amdgpu_virt_mmio_blocked(adev) &&
3866                     !amdgpu_virt_wait_reset(adev)) {
3867                         dev_err(adev->dev, "VF exclusive mode timeout\n");
3868                         /* Don't send request since VF is inactive. */
3869                         adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
3870                         adev->virt.ops = NULL;
3871                         r = -EAGAIN;
3872                         goto release_ras_con;
3873                 }
3874                 dev_err(adev->dev, "amdgpu_device_ip_init failed\n");
3875                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_INIT_FAIL, 0, 0);
3876                 goto release_ras_con;
3877         }
3878
3879         amdgpu_fence_driver_hw_init(adev);
3880
3881         dev_info(adev->dev,
3882                 "SE %d, SH per SE %d, CU per SH %d, active_cu_number %d\n",
3883                         adev->gfx.config.max_shader_engines,
3884                         adev->gfx.config.max_sh_per_se,
3885                         adev->gfx.config.max_cu_per_sh,
3886                         adev->gfx.cu_info.number);
3887
3888         adev->accel_working = true;
3889
3890         amdgpu_vm_check_compute_bug(adev);
3891
3892         /* Initialize the buffer migration limit. */
3893         if (amdgpu_moverate >= 0)
3894                 max_MBps = amdgpu_moverate;
3895         else
3896                 max_MBps = 8; /* Allow 8 MB/s. */
3897         /* Get a log2 for easy divisions. */
3898         adev->mm_stats.log2_max_MBps = ilog2(max(1u, max_MBps));
3899
3900         r = amdgpu_pm_sysfs_init(adev);
3901         if (r)
3902                 DRM_ERROR("registering pm sysfs failed (%d).\n", r);
3903
3904         r = amdgpu_ucode_sysfs_init(adev);
3905         if (r) {
3906                 adev->ucode_sysfs_en = false;
3907                 DRM_ERROR("Creating firmware sysfs failed (%d).\n", r);
3908         } else
3909                 adev->ucode_sysfs_en = true;
3910
3911         r = amdgpu_psp_sysfs_init(adev);
3912         if (r) {
3913                 adev->psp_sysfs_en = false;
3914                 if (!amdgpu_sriov_vf(adev))
3915                         DRM_ERROR("Creating psp sysfs failed\n");
3916         } else
3917                 adev->psp_sysfs_en = true;
3918
3919         /*
3920          * Register gpu instance before amdgpu_device_enable_mgpu_fan_boost.
3921          * Otherwise the mgpu fan boost feature will be skipped due to the
3922          * gpu instance is counted less.
3923          */
3924         amdgpu_register_gpu_instance(adev);
3925
3926         /* enable clockgating, etc. after ib tests, etc. since some blocks require
3927          * explicit gating rather than handling it automatically.
3928          */
3929         if (!adev->gmc.xgmi.pending_reset) {
3930                 r = amdgpu_device_ip_late_init(adev);
3931                 if (r) {
3932                         dev_err(adev->dev, "amdgpu_device_ip_late_init failed\n");
3933                         amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_LATE_INIT_FAIL, 0, r);
3934                         goto release_ras_con;
3935                 }
3936                 /* must succeed. */
3937                 amdgpu_ras_resume(adev);
3938                 queue_delayed_work(system_wq, &adev->delayed_init_work,
3939                                    msecs_to_jiffies(AMDGPU_RESUME_MS));
3940         }
3941
3942         if (amdgpu_sriov_vf(adev))
3943                 flush_delayed_work(&adev->delayed_init_work);
3944
3945         r = sysfs_create_files(&adev->dev->kobj, amdgpu_dev_attributes);
3946         if (r)
3947                 dev_err(adev->dev, "Could not create amdgpu device attr\n");
3948
3949         if (IS_ENABLED(CONFIG_PERF_EVENTS))
3950                 r = amdgpu_pmu_init(adev);
3951         if (r)
3952                 dev_err(adev->dev, "amdgpu_pmu_init failed\n");
3953
3954         /* Have stored pci confspace at hand for restore in sudden PCI error */
3955         if (amdgpu_device_cache_pci_state(adev->pdev))
3956                 pci_restore_state(pdev);
3957
3958         /* if we have > 1 VGA cards, then disable the amdgpu VGA resources */
3959         /* this will fail for cards that aren't VGA class devices, just
3960          * ignore it */
3961         if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA)
3962                 vga_client_register(adev->pdev, amdgpu_device_vga_set_decode);
3963
3964         px = amdgpu_device_supports_px(ddev);
3965
3966         if (px || (!pci_is_thunderbolt_attached(adev->pdev) &&
3967                                 apple_gmux_detect(NULL, NULL)))
3968                 vga_switcheroo_register_client(adev->pdev,
3969                                                &amdgpu_switcheroo_ops, px);
3970
3971         if (px)
3972                 vga_switcheroo_init_domain_pm_ops(adev->dev, &adev->vga_pm_domain);
3973
3974         if (adev->gmc.xgmi.pending_reset)
3975                 queue_delayed_work(system_wq, &mgpu_info.delayed_reset_work,
3976                                    msecs_to_jiffies(AMDGPU_RESUME_MS));
3977
3978         amdgpu_device_check_iommu_direct_map(adev);
3979
3980         return 0;
3981
3982 release_ras_con:
3983         amdgpu_release_ras_context(adev);
3984
3985 failed:
3986         amdgpu_vf_error_trans_all(adev);
3987
3988         return r;
3989 }
3990
3991 static void amdgpu_device_unmap_mmio(struct amdgpu_device *adev)
3992 {
3993
3994         /* Clear all CPU mappings pointing to this device */
3995         unmap_mapping_range(adev->ddev.anon_inode->i_mapping, 0, 0, 1);
3996
3997         /* Unmap all mapped bars - Doorbell, registers and VRAM */
3998         amdgpu_device_doorbell_fini(adev);
3999
4000         iounmap(adev->rmmio);
4001         adev->rmmio = NULL;
4002         if (adev->mman.aper_base_kaddr)
4003                 iounmap(adev->mman.aper_base_kaddr);
4004         adev->mman.aper_base_kaddr = NULL;
4005
4006         /* Memory manager related */
4007         if (!adev->gmc.xgmi.connected_to_cpu) {
4008                 arch_phys_wc_del(adev->gmc.vram_mtrr);
4009                 arch_io_free_memtype_wc(adev->gmc.aper_base, adev->gmc.aper_size);
4010         }
4011 }
4012
4013 /**
4014  * amdgpu_device_fini_hw - tear down the driver
4015  *
4016  * @adev: amdgpu_device pointer
4017  *
4018  * Tear down the driver info (all asics).
4019  * Called at driver shutdown.
4020  */
4021 void amdgpu_device_fini_hw(struct amdgpu_device *adev)
4022 {
4023         dev_info(adev->dev, "amdgpu: finishing device.\n");
4024         flush_delayed_work(&adev->delayed_init_work);
4025         adev->shutdown = true;
4026
4027         /* make sure IB test finished before entering exclusive mode
4028          * to avoid preemption on IB test
4029          * */
4030         if (amdgpu_sriov_vf(adev)) {
4031                 amdgpu_virt_request_full_gpu(adev, false);
4032                 amdgpu_virt_fini_data_exchange(adev);
4033         }
4034
4035         /* disable all interrupts */
4036         amdgpu_irq_disable_all(adev);
4037         if (adev->mode_info.mode_config_initialized){
4038                 if (!drm_drv_uses_atomic_modeset(adev_to_drm(adev)))
4039                         drm_helper_force_disable_all(adev_to_drm(adev));
4040                 else
4041                         drm_atomic_helper_shutdown(adev_to_drm(adev));
4042         }
4043         amdgpu_fence_driver_hw_fini(adev);
4044
4045         if (adev->mman.initialized)
4046                 drain_workqueue(adev->mman.bdev.wq);
4047
4048         if (adev->pm.sysfs_initialized)
4049                 amdgpu_pm_sysfs_fini(adev);
4050         if (adev->ucode_sysfs_en)
4051                 amdgpu_ucode_sysfs_fini(adev);
4052         if (adev->psp_sysfs_en)
4053                 amdgpu_psp_sysfs_fini(adev);
4054         sysfs_remove_files(&adev->dev->kobj, amdgpu_dev_attributes);
4055
4056         /* disable ras feature must before hw fini */
4057         amdgpu_ras_pre_fini(adev);
4058
4059         amdgpu_device_ip_fini_early(adev);
4060
4061         amdgpu_irq_fini_hw(adev);
4062
4063         if (adev->mman.initialized)
4064                 ttm_device_clear_dma_mappings(&adev->mman.bdev);
4065
4066         amdgpu_gart_dummy_page_fini(adev);
4067
4068         if (drm_dev_is_unplugged(adev_to_drm(adev)))
4069                 amdgpu_device_unmap_mmio(adev);
4070
4071 }
4072
4073 void amdgpu_device_fini_sw(struct amdgpu_device *adev)
4074 {
4075         int idx;
4076         bool px;
4077
4078         amdgpu_fence_driver_sw_fini(adev);
4079         amdgpu_device_ip_fini(adev);
4080         amdgpu_ucode_release(&adev->firmware.gpu_info_fw);
4081         adev->accel_working = false;
4082         dma_fence_put(rcu_dereference_protected(adev->gang_submit, true));
4083
4084         amdgpu_reset_fini(adev);
4085
4086         /* free i2c buses */
4087         if (!amdgpu_device_has_dc_support(adev))
4088                 amdgpu_i2c_fini(adev);
4089
4090         if (amdgpu_emu_mode != 1)
4091                 amdgpu_atombios_fini(adev);
4092
4093         kfree(adev->bios);
4094         adev->bios = NULL;
4095
4096         px = amdgpu_device_supports_px(adev_to_drm(adev));
4097
4098         if (px || (!pci_is_thunderbolt_attached(adev->pdev) &&
4099                                 apple_gmux_detect(NULL, NULL)))
4100                 vga_switcheroo_unregister_client(adev->pdev);
4101
4102         if (px)
4103                 vga_switcheroo_fini_domain_pm_ops(adev->dev);
4104
4105         if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA)
4106                 vga_client_unregister(adev->pdev);
4107
4108         if (drm_dev_enter(adev_to_drm(adev), &idx)) {
4109
4110                 iounmap(adev->rmmio);
4111                 adev->rmmio = NULL;
4112                 amdgpu_device_doorbell_fini(adev);
4113                 drm_dev_exit(idx);
4114         }
4115
4116         if (IS_ENABLED(CONFIG_PERF_EVENTS))
4117                 amdgpu_pmu_fini(adev);
4118         if (adev->mman.discovery_bin)
4119                 amdgpu_discovery_fini(adev);
4120
4121         amdgpu_reset_put_reset_domain(adev->reset_domain);
4122         adev->reset_domain = NULL;
4123
4124         kfree(adev->pci_state);
4125
4126 }
4127
4128 /**
4129  * amdgpu_device_evict_resources - evict device resources
4130  * @adev: amdgpu device object
4131  *
4132  * Evicts all ttm device resources(vram BOs, gart table) from the lru list
4133  * of the vram memory type. Mainly used for evicting device resources
4134  * at suspend time.
4135  *
4136  */
4137 static int amdgpu_device_evict_resources(struct amdgpu_device *adev)
4138 {
4139         int ret;
4140
4141         /* No need to evict vram on APUs for suspend to ram or s2idle */
4142         if ((adev->in_s3 || adev->in_s0ix) && (adev->flags & AMD_IS_APU))
4143                 return 0;
4144
4145         ret = amdgpu_ttm_evict_resources(adev, TTM_PL_VRAM);
4146         if (ret)
4147                 DRM_WARN("evicting device resources failed\n");
4148         return ret;
4149 }
4150
4151 /*
4152  * Suspend & resume.
4153  */
4154 /**
4155  * amdgpu_device_suspend - initiate device suspend
4156  *
4157  * @dev: drm dev pointer
4158  * @fbcon : notify the fbdev of suspend
4159  *
4160  * Puts the hw in the suspend state (all asics).
4161  * Returns 0 for success or an error on failure.
4162  * Called at driver suspend.
4163  */
4164 int amdgpu_device_suspend(struct drm_device *dev, bool fbcon)
4165 {
4166         struct amdgpu_device *adev = drm_to_adev(dev);
4167         int r = 0;
4168
4169         if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
4170                 return 0;
4171
4172         adev->in_suspend = true;
4173
4174         /* Evict the majority of BOs before grabbing the full access */
4175         r = amdgpu_device_evict_resources(adev);
4176         if (r)
4177                 return r;
4178
4179         if (amdgpu_sriov_vf(adev)) {
4180                 amdgpu_virt_fini_data_exchange(adev);
4181                 r = amdgpu_virt_request_full_gpu(adev, false);
4182                 if (r)
4183                         return r;
4184         }
4185
4186         if (amdgpu_acpi_smart_shift_update(dev, AMDGPU_SS_DEV_D3))
4187                 DRM_WARN("smart shift update failed\n");
4188
4189         if (fbcon)
4190                 drm_fb_helper_set_suspend_unlocked(adev_to_drm(adev)->fb_helper, true);
4191
4192         cancel_delayed_work_sync(&adev->delayed_init_work);
4193
4194         amdgpu_ras_suspend(adev);
4195
4196         amdgpu_device_ip_suspend_phase1(adev);
4197
4198         if (!adev->in_s0ix)
4199                 amdgpu_amdkfd_suspend(adev, adev->in_runpm);
4200
4201         r = amdgpu_device_evict_resources(adev);
4202         if (r)
4203                 return r;
4204
4205         amdgpu_fence_driver_hw_fini(adev);
4206
4207         amdgpu_device_ip_suspend_phase2(adev);
4208
4209         if (amdgpu_sriov_vf(adev))
4210                 amdgpu_virt_release_full_gpu(adev, false);
4211
4212         return 0;
4213 }
4214
4215 /**
4216  * amdgpu_device_resume - initiate device resume
4217  *
4218  * @dev: drm dev pointer
4219  * @fbcon : notify the fbdev of resume
4220  *
4221  * Bring the hw back to operating state (all asics).
4222  * Returns 0 for success or an error on failure.
4223  * Called at driver resume.
4224  */
4225 int amdgpu_device_resume(struct drm_device *dev, bool fbcon)
4226 {
4227         struct amdgpu_device *adev = drm_to_adev(dev);
4228         int r = 0;
4229
4230         if (amdgpu_sriov_vf(adev)) {
4231                 r = amdgpu_virt_request_full_gpu(adev, true);
4232                 if (r)
4233                         return r;
4234         }
4235
4236         if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
4237                 return 0;
4238
4239         if (adev->in_s0ix)
4240                 amdgpu_dpm_gfx_state_change(adev, sGpuChangeState_D0Entry);
4241
4242         /* post card */
4243         if (amdgpu_device_need_post(adev)) {
4244                 r = amdgpu_device_asic_init(adev);
4245                 if (r)
4246                         dev_err(adev->dev, "amdgpu asic init failed\n");
4247         }
4248
4249         r = amdgpu_device_ip_resume(adev);
4250
4251         if (r) {
4252                 dev_err(adev->dev, "amdgpu_device_ip_resume failed (%d).\n", r);
4253                 goto exit;
4254         }
4255         amdgpu_fence_driver_hw_init(adev);
4256
4257         r = amdgpu_device_ip_late_init(adev);
4258         if (r)
4259                 goto exit;
4260
4261         queue_delayed_work(system_wq, &adev->delayed_init_work,
4262                            msecs_to_jiffies(AMDGPU_RESUME_MS));
4263
4264         if (!adev->in_s0ix) {
4265                 r = amdgpu_amdkfd_resume(adev, adev->in_runpm);
4266                 if (r)
4267                         goto exit;
4268         }
4269
4270 exit:
4271         if (amdgpu_sriov_vf(adev)) {
4272                 amdgpu_virt_init_data_exchange(adev);
4273                 amdgpu_virt_release_full_gpu(adev, true);
4274         }
4275
4276         if (r)
4277                 return r;
4278
4279         /* Make sure IB tests flushed */
4280         flush_delayed_work(&adev->delayed_init_work);
4281
4282         if (fbcon)
4283                 drm_fb_helper_set_suspend_unlocked(adev_to_drm(adev)->fb_helper, false);
4284
4285         amdgpu_ras_resume(adev);
4286
4287         if (adev->mode_info.num_crtc) {
4288                 /*
4289                  * Most of the connector probing functions try to acquire runtime pm
4290                  * refs to ensure that the GPU is powered on when connector polling is
4291                  * performed. Since we're calling this from a runtime PM callback,
4292                  * trying to acquire rpm refs will cause us to deadlock.
4293                  *
4294                  * Since we're guaranteed to be holding the rpm lock, it's safe to
4295                  * temporarily disable the rpm helpers so this doesn't deadlock us.
4296                  */
4297 #ifdef CONFIG_PM
4298                 dev->dev->power.disable_depth++;
4299 #endif
4300                 if (!adev->dc_enabled)
4301                         drm_helper_hpd_irq_event(dev);
4302                 else
4303                         drm_kms_helper_hotplug_event(dev);
4304 #ifdef CONFIG_PM
4305                 dev->dev->power.disable_depth--;
4306 #endif
4307         }
4308         adev->in_suspend = false;
4309
4310         if (adev->enable_mes)
4311                 amdgpu_mes_self_test(adev);
4312
4313         if (amdgpu_acpi_smart_shift_update(dev, AMDGPU_SS_DEV_D0))
4314                 DRM_WARN("smart shift update failed\n");
4315
4316         return 0;
4317 }
4318
4319 /**
4320  * amdgpu_device_ip_check_soft_reset - did soft reset succeed
4321  *
4322  * @adev: amdgpu_device pointer
4323  *
4324  * The list of all the hardware IPs that make up the asic is walked and
4325  * the check_soft_reset callbacks are run.  check_soft_reset determines
4326  * if the asic is still hung or not.
4327  * Returns true if any of the IPs are still in a hung state, false if not.
4328  */
4329 static bool amdgpu_device_ip_check_soft_reset(struct amdgpu_device *adev)
4330 {
4331         int i;
4332         bool asic_hang = false;
4333
4334         if (amdgpu_sriov_vf(adev))
4335                 return true;
4336
4337         if (amdgpu_asic_need_full_reset(adev))
4338                 return true;
4339
4340         for (i = 0; i < adev->num_ip_blocks; i++) {
4341                 if (!adev->ip_blocks[i].status.valid)
4342                         continue;
4343                 if (adev->ip_blocks[i].version->funcs->check_soft_reset)
4344                         adev->ip_blocks[i].status.hang =
4345                                 adev->ip_blocks[i].version->funcs->check_soft_reset(adev);
4346                 if (adev->ip_blocks[i].status.hang) {
4347                         dev_info(adev->dev, "IP block:%s is hung!\n", adev->ip_blocks[i].version->funcs->name);
4348                         asic_hang = true;
4349                 }
4350         }
4351         return asic_hang;
4352 }
4353
4354 /**
4355  * amdgpu_device_ip_pre_soft_reset - prepare for soft reset
4356  *
4357  * @adev: amdgpu_device pointer
4358  *
4359  * The list of all the hardware IPs that make up the asic is walked and the
4360  * pre_soft_reset callbacks are run if the block is hung.  pre_soft_reset
4361  * handles any IP specific hardware or software state changes that are
4362  * necessary for a soft reset to succeed.
4363  * Returns 0 on success, negative error code on failure.
4364  */
4365 static int amdgpu_device_ip_pre_soft_reset(struct amdgpu_device *adev)
4366 {
4367         int i, r = 0;
4368
4369         for (i = 0; i < adev->num_ip_blocks; i++) {
4370                 if (!adev->ip_blocks[i].status.valid)
4371                         continue;
4372                 if (adev->ip_blocks[i].status.hang &&
4373                     adev->ip_blocks[i].version->funcs->pre_soft_reset) {
4374                         r = adev->ip_blocks[i].version->funcs->pre_soft_reset(adev);
4375                         if (r)
4376                                 return r;
4377                 }
4378         }
4379
4380         return 0;
4381 }
4382
4383 /**
4384  * amdgpu_device_ip_need_full_reset - check if a full asic reset is needed
4385  *
4386  * @adev: amdgpu_device pointer
4387  *
4388  * Some hardware IPs cannot be soft reset.  If they are hung, a full gpu
4389  * reset is necessary to recover.
4390  * Returns true if a full asic reset is required, false if not.
4391  */
4392 static bool amdgpu_device_ip_need_full_reset(struct amdgpu_device *adev)
4393 {
4394         int i;
4395
4396         if (amdgpu_asic_need_full_reset(adev))
4397                 return true;
4398
4399         for (i = 0; i < adev->num_ip_blocks; i++) {
4400                 if (!adev->ip_blocks[i].status.valid)
4401                         continue;
4402                 if ((adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) ||
4403                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) ||
4404                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_ACP) ||
4405                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) ||
4406                      adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) {
4407                         if (adev->ip_blocks[i].status.hang) {
4408                                 dev_info(adev->dev, "Some block need full reset!\n");
4409                                 return true;
4410                         }
4411                 }
4412         }
4413         return false;
4414 }
4415
4416 /**
4417  * amdgpu_device_ip_soft_reset - do a soft reset
4418  *
4419  * @adev: amdgpu_device pointer
4420  *
4421  * The list of all the hardware IPs that make up the asic is walked and the
4422  * soft_reset callbacks are run if the block is hung.  soft_reset handles any
4423  * IP specific hardware or software state changes that are necessary to soft
4424  * reset the IP.
4425  * Returns 0 on success, negative error code on failure.
4426  */
4427 static int amdgpu_device_ip_soft_reset(struct amdgpu_device *adev)
4428 {
4429         int i, r = 0;
4430
4431         for (i = 0; i < adev->num_ip_blocks; i++) {
4432                 if (!adev->ip_blocks[i].status.valid)
4433                         continue;
4434                 if (adev->ip_blocks[i].status.hang &&
4435                     adev->ip_blocks[i].version->funcs->soft_reset) {
4436                         r = adev->ip_blocks[i].version->funcs->soft_reset(adev);
4437                         if (r)
4438                                 return r;
4439                 }
4440         }
4441
4442         return 0;
4443 }
4444
4445 /**
4446  * amdgpu_device_ip_post_soft_reset - clean up from soft reset
4447  *
4448  * @adev: amdgpu_device pointer
4449  *
4450  * The list of all the hardware IPs that make up the asic is walked and the
4451  * post_soft_reset callbacks are run if the asic was hung.  post_soft_reset
4452  * handles any IP specific hardware or software state changes that are
4453  * necessary after the IP has been soft reset.
4454  * Returns 0 on success, negative error code on failure.
4455  */
4456 static int amdgpu_device_ip_post_soft_reset(struct amdgpu_device *adev)
4457 {
4458         int i, r = 0;
4459
4460         for (i = 0; i < adev->num_ip_blocks; i++) {
4461                 if (!adev->ip_blocks[i].status.valid)
4462                         continue;
4463                 if (adev->ip_blocks[i].status.hang &&
4464                     adev->ip_blocks[i].version->funcs->post_soft_reset)
4465                         r = adev->ip_blocks[i].version->funcs->post_soft_reset(adev);
4466                 if (r)
4467                         return r;
4468         }
4469
4470         return 0;
4471 }
4472
4473 /**
4474  * amdgpu_device_recover_vram - Recover some VRAM contents
4475  *
4476  * @adev: amdgpu_device pointer
4477  *
4478  * Restores the contents of VRAM buffers from the shadows in GTT.  Used to
4479  * restore things like GPUVM page tables after a GPU reset where
4480  * the contents of VRAM might be lost.
4481  *
4482  * Returns:
4483  * 0 on success, negative error code on failure.
4484  */
4485 static int amdgpu_device_recover_vram(struct amdgpu_device *adev)
4486 {
4487         struct dma_fence *fence = NULL, *next = NULL;
4488         struct amdgpu_bo *shadow;
4489         struct amdgpu_bo_vm *vmbo;
4490         long r = 1, tmo;
4491
4492         if (amdgpu_sriov_runtime(adev))
4493                 tmo = msecs_to_jiffies(8000);
4494         else
4495                 tmo = msecs_to_jiffies(100);
4496
4497         dev_info(adev->dev, "recover vram bo from shadow start\n");
4498         mutex_lock(&adev->shadow_list_lock);
4499         list_for_each_entry(vmbo, &adev->shadow_list, shadow_list) {
4500                 shadow = &vmbo->bo;
4501                 /* No need to recover an evicted BO */
4502                 if (shadow->tbo.resource->mem_type != TTM_PL_TT ||
4503                     shadow->tbo.resource->start == AMDGPU_BO_INVALID_OFFSET ||
4504                     shadow->parent->tbo.resource->mem_type != TTM_PL_VRAM)
4505                         continue;
4506
4507                 r = amdgpu_bo_restore_shadow(shadow, &next);
4508                 if (r)
4509                         break;
4510
4511                 if (fence) {
4512                         tmo = dma_fence_wait_timeout(fence, false, tmo);
4513                         dma_fence_put(fence);
4514                         fence = next;
4515                         if (tmo == 0) {
4516                                 r = -ETIMEDOUT;
4517                                 break;
4518                         } else if (tmo < 0) {
4519                                 r = tmo;
4520                                 break;
4521                         }
4522                 } else {
4523                         fence = next;
4524                 }
4525         }
4526         mutex_unlock(&adev->shadow_list_lock);
4527
4528         if (fence)
4529                 tmo = dma_fence_wait_timeout(fence, false, tmo);
4530         dma_fence_put(fence);
4531
4532         if (r < 0 || tmo <= 0) {
4533                 dev_err(adev->dev, "recover vram bo from shadow failed, r is %ld, tmo is %ld\n", r, tmo);
4534                 return -EIO;
4535         }
4536
4537         dev_info(adev->dev, "recover vram bo from shadow done\n");
4538         return 0;
4539 }
4540
4541
4542 /**
4543  * amdgpu_device_reset_sriov - reset ASIC for SR-IOV vf
4544  *
4545  * @adev: amdgpu_device pointer
4546  * @from_hypervisor: request from hypervisor
4547  *
4548  * do VF FLR and reinitialize Asic
4549  * return 0 means succeeded otherwise failed
4550  */
4551 static int amdgpu_device_reset_sriov(struct amdgpu_device *adev,
4552                                      bool from_hypervisor)
4553 {
4554         int r;
4555         struct amdgpu_hive_info *hive = NULL;
4556         int retry_limit = 0;
4557
4558 retry:
4559         amdgpu_amdkfd_pre_reset(adev);
4560
4561         if (from_hypervisor)
4562                 r = amdgpu_virt_request_full_gpu(adev, true);
4563         else
4564                 r = amdgpu_virt_reset_gpu(adev);
4565         if (r)
4566                 return r;
4567
4568         /* Resume IP prior to SMC */
4569         r = amdgpu_device_ip_reinit_early_sriov(adev);
4570         if (r)
4571                 goto error;
4572
4573         amdgpu_virt_init_data_exchange(adev);
4574
4575         r = amdgpu_device_fw_loading(adev);
4576         if (r)
4577                 return r;
4578
4579         /* now we are okay to resume SMC/CP/SDMA */
4580         r = amdgpu_device_ip_reinit_late_sriov(adev);
4581         if (r)
4582                 goto error;
4583
4584         hive = amdgpu_get_xgmi_hive(adev);
4585         /* Update PSP FW topology after reset */
4586         if (hive && adev->gmc.xgmi.num_physical_nodes > 1)
4587                 r = amdgpu_xgmi_update_topology(hive, adev);
4588
4589         if (hive)
4590                 amdgpu_put_xgmi_hive(hive);
4591
4592         if (!r) {
4593                 amdgpu_irq_gpu_reset_resume_helper(adev);
4594                 r = amdgpu_ib_ring_tests(adev);
4595
4596                 amdgpu_amdkfd_post_reset(adev);
4597         }
4598
4599 error:
4600         if (!r && adev->virt.gim_feature & AMDGIM_FEATURE_GIM_FLR_VRAMLOST) {
4601                 amdgpu_inc_vram_lost(adev);
4602                 r = amdgpu_device_recover_vram(adev);
4603         }
4604         amdgpu_virt_release_full_gpu(adev, true);
4605
4606         if (AMDGPU_RETRY_SRIOV_RESET(r)) {
4607                 if (retry_limit < AMDGPU_MAX_RETRY_LIMIT) {
4608                         retry_limit++;
4609                         goto retry;
4610                 } else
4611                         DRM_ERROR("GPU reset retry is beyond the retry limit\n");
4612         }
4613
4614         return r;
4615 }
4616
4617 /**
4618  * amdgpu_device_has_job_running - check if there is any job in mirror list
4619  *
4620  * @adev: amdgpu_device pointer
4621  *
4622  * check if there is any job in mirror list
4623  */
4624 bool amdgpu_device_has_job_running(struct amdgpu_device *adev)
4625 {
4626         int i;
4627         struct drm_sched_job *job;
4628
4629         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
4630                 struct amdgpu_ring *ring = adev->rings[i];
4631
4632                 if (!ring || !ring->sched.thread)
4633                         continue;
4634
4635                 spin_lock(&ring->sched.job_list_lock);
4636                 job = list_first_entry_or_null(&ring->sched.pending_list,
4637                                                struct drm_sched_job, list);
4638                 spin_unlock(&ring->sched.job_list_lock);
4639                 if (job)
4640                         return true;
4641         }
4642         return false;
4643 }
4644
4645 /**
4646  * amdgpu_device_should_recover_gpu - check if we should try GPU recovery
4647  *
4648  * @adev: amdgpu_device pointer
4649  *
4650  * Check amdgpu_gpu_recovery and SRIOV status to see if we should try to recover
4651  * a hung GPU.
4652  */
4653 bool amdgpu_device_should_recover_gpu(struct amdgpu_device *adev)
4654 {
4655
4656         if (amdgpu_gpu_recovery == 0)
4657                 goto disabled;
4658
4659         /* Skip soft reset check in fatal error mode */
4660         if (!amdgpu_ras_is_poison_mode_supported(adev))
4661                 return true;
4662
4663         if (amdgpu_sriov_vf(adev))
4664                 return true;
4665
4666         if (amdgpu_gpu_recovery == -1) {
4667                 switch (adev->asic_type) {
4668 #ifdef CONFIG_DRM_AMDGPU_SI
4669                 case CHIP_VERDE:
4670                 case CHIP_TAHITI:
4671                 case CHIP_PITCAIRN:
4672                 case CHIP_OLAND:
4673                 case CHIP_HAINAN:
4674 #endif
4675 #ifdef CONFIG_DRM_AMDGPU_CIK
4676                 case CHIP_KAVERI:
4677                 case CHIP_KABINI:
4678                 case CHIP_MULLINS:
4679 #endif
4680                 case CHIP_CARRIZO:
4681                 case CHIP_STONEY:
4682                 case CHIP_CYAN_SKILLFISH:
4683                         goto disabled;
4684                 default:
4685                         break;
4686                 }
4687         }
4688
4689         return true;
4690
4691 disabled:
4692                 dev_info(adev->dev, "GPU recovery disabled.\n");
4693                 return false;
4694 }
4695
4696 int amdgpu_device_mode1_reset(struct amdgpu_device *adev)
4697 {
4698         u32 i;
4699         int ret = 0;
4700
4701         amdgpu_atombios_scratch_regs_engine_hung(adev, true);
4702
4703         dev_info(adev->dev, "GPU mode1 reset\n");
4704
4705         /* disable BM */
4706         pci_clear_master(adev->pdev);
4707
4708         amdgpu_device_cache_pci_state(adev->pdev);
4709
4710         if (amdgpu_dpm_is_mode1_reset_supported(adev)) {
4711                 dev_info(adev->dev, "GPU smu mode1 reset\n");
4712                 ret = amdgpu_dpm_mode1_reset(adev);
4713         } else {
4714                 dev_info(adev->dev, "GPU psp mode1 reset\n");
4715                 ret = psp_gpu_reset(adev);
4716         }
4717
4718         if (ret)
4719                 dev_err(adev->dev, "GPU mode1 reset failed\n");
4720
4721         amdgpu_device_load_pci_state(adev->pdev);
4722
4723         /* wait for asic to come out of reset */
4724         for (i = 0; i < adev->usec_timeout; i++) {
4725                 u32 memsize = adev->nbio.funcs->get_memsize(adev);
4726
4727                 if (memsize != 0xffffffff)
4728                         break;
4729                 udelay(1);
4730         }
4731
4732         amdgpu_atombios_scratch_regs_engine_hung(adev, false);
4733         return ret;
4734 }
4735
4736 int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev,
4737                                  struct amdgpu_reset_context *reset_context)
4738 {
4739         int i, r = 0;
4740         struct amdgpu_job *job = NULL;
4741         bool need_full_reset =
4742                 test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
4743
4744         if (reset_context->reset_req_dev == adev)
4745                 job = reset_context->job;
4746
4747         if (amdgpu_sriov_vf(adev)) {
4748                 /* stop the data exchange thread */
4749                 amdgpu_virt_fini_data_exchange(adev);
4750         }
4751
4752         amdgpu_fence_driver_isr_toggle(adev, true);
4753
4754         /* block all schedulers and reset given job's ring */
4755         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
4756                 struct amdgpu_ring *ring = adev->rings[i];
4757
4758                 if (!ring || !ring->sched.thread)
4759                         continue;
4760
4761                 /*clear job fence from fence drv to avoid force_completion
4762                  *leave NULL and vm flush fence in fence drv */
4763                 amdgpu_fence_driver_clear_job_fences(ring);
4764
4765                 /* after all hw jobs are reset, hw fence is meaningless, so force_completion */
4766                 amdgpu_fence_driver_force_completion(ring);
4767         }
4768
4769         amdgpu_fence_driver_isr_toggle(adev, false);
4770
4771         if (job && job->vm)
4772                 drm_sched_increase_karma(&job->base);
4773
4774         r = amdgpu_reset_prepare_hwcontext(adev, reset_context);
4775         /* If reset handler not implemented, continue; otherwise return */
4776         if (r == -ENOSYS)
4777                 r = 0;
4778         else
4779                 return r;
4780
4781         /* Don't suspend on bare metal if we are not going to HW reset the ASIC */
4782         if (!amdgpu_sriov_vf(adev)) {
4783
4784                 if (!need_full_reset)
4785                         need_full_reset = amdgpu_device_ip_need_full_reset(adev);
4786
4787                 if (!need_full_reset && amdgpu_gpu_recovery &&
4788                     amdgpu_device_ip_check_soft_reset(adev)) {
4789                         amdgpu_device_ip_pre_soft_reset(adev);
4790                         r = amdgpu_device_ip_soft_reset(adev);
4791                         amdgpu_device_ip_post_soft_reset(adev);
4792                         if (r || amdgpu_device_ip_check_soft_reset(adev)) {
4793                                 dev_info(adev->dev, "soft reset failed, will fallback to full reset!\n");
4794                                 need_full_reset = true;
4795                         }
4796                 }
4797
4798                 if (need_full_reset)
4799                         r = amdgpu_device_ip_suspend(adev);
4800                 if (need_full_reset)
4801                         set_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
4802                 else
4803                         clear_bit(AMDGPU_NEED_FULL_RESET,
4804                                   &reset_context->flags);
4805         }
4806
4807         return r;
4808 }
4809
4810 static int amdgpu_reset_reg_dumps(struct amdgpu_device *adev)
4811 {
4812         int i;
4813
4814         lockdep_assert_held(&adev->reset_domain->sem);
4815
4816         for (i = 0; i < adev->num_regs; i++) {
4817                 adev->reset_dump_reg_value[i] = RREG32(adev->reset_dump_reg_list[i]);
4818                 trace_amdgpu_reset_reg_dumps(adev->reset_dump_reg_list[i],
4819                                              adev->reset_dump_reg_value[i]);
4820         }
4821
4822         return 0;
4823 }
4824
4825 #ifdef CONFIG_DEV_COREDUMP
4826 static ssize_t amdgpu_devcoredump_read(char *buffer, loff_t offset,
4827                 size_t count, void *data, size_t datalen)
4828 {
4829         struct drm_printer p;
4830         struct amdgpu_device *adev = data;
4831         struct drm_print_iterator iter;
4832         int i;
4833
4834         iter.data = buffer;
4835         iter.offset = 0;
4836         iter.start = offset;
4837         iter.remain = count;
4838
4839         p = drm_coredump_printer(&iter);
4840
4841         drm_printf(&p, "**** AMDGPU Device Coredump ****\n");
4842         drm_printf(&p, "kernel: " UTS_RELEASE "\n");
4843         drm_printf(&p, "module: " KBUILD_MODNAME "\n");
4844         drm_printf(&p, "time: %lld.%09ld\n", adev->reset_time.tv_sec, adev->reset_time.tv_nsec);
4845         if (adev->reset_task_info.pid)
4846                 drm_printf(&p, "process_name: %s PID: %d\n",
4847                            adev->reset_task_info.process_name,
4848                            adev->reset_task_info.pid);
4849
4850         if (adev->reset_vram_lost)
4851                 drm_printf(&p, "VRAM is lost due to GPU reset!\n");
4852         if (adev->num_regs) {
4853                 drm_printf(&p, "AMDGPU register dumps:\nOffset:     Value:\n");
4854
4855                 for (i = 0; i < adev->num_regs; i++)
4856                         drm_printf(&p, "0x%08x: 0x%08x\n",
4857                                    adev->reset_dump_reg_list[i],
4858                                    adev->reset_dump_reg_value[i]);
4859         }
4860
4861         return count - iter.remain;
4862 }
4863
4864 static void amdgpu_devcoredump_free(void *data)
4865 {
4866 }
4867
4868 static void amdgpu_reset_capture_coredumpm(struct amdgpu_device *adev)
4869 {
4870         struct drm_device *dev = adev_to_drm(adev);
4871
4872         ktime_get_ts64(&adev->reset_time);
4873         dev_coredumpm(dev->dev, THIS_MODULE, adev, 0, GFP_KERNEL,
4874                       amdgpu_devcoredump_read, amdgpu_devcoredump_free);
4875 }
4876 #endif
4877
4878 int amdgpu_do_asic_reset(struct list_head *device_list_handle,
4879                          struct amdgpu_reset_context *reset_context)
4880 {
4881         struct amdgpu_device *tmp_adev = NULL;
4882         bool need_full_reset, skip_hw_reset, vram_lost = false;
4883         int r = 0;
4884         bool gpu_reset_for_dev_remove = 0;
4885
4886         /* Try reset handler method first */
4887         tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device,
4888                                     reset_list);
4889         amdgpu_reset_reg_dumps(tmp_adev);
4890
4891         reset_context->reset_device_list = device_list_handle;
4892         r = amdgpu_reset_perform_reset(tmp_adev, reset_context);
4893         /* If reset handler not implemented, continue; otherwise return */
4894         if (r == -ENOSYS)
4895                 r = 0;
4896         else
4897                 return r;
4898
4899         /* Reset handler not implemented, use the default method */
4900         need_full_reset =
4901                 test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
4902         skip_hw_reset = test_bit(AMDGPU_SKIP_HW_RESET, &reset_context->flags);
4903
4904         gpu_reset_for_dev_remove =
4905                 test_bit(AMDGPU_RESET_FOR_DEVICE_REMOVE, &reset_context->flags) &&
4906                         test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
4907
4908         /*
4909          * ASIC reset has to be done on all XGMI hive nodes ASAP
4910          * to allow proper links negotiation in FW (within 1 sec)
4911          */
4912         if (!skip_hw_reset && need_full_reset) {
4913                 list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
4914                         /* For XGMI run all resets in parallel to speed up the process */
4915                         if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) {
4916                                 tmp_adev->gmc.xgmi.pending_reset = false;
4917                                 if (!queue_work(system_unbound_wq, &tmp_adev->xgmi_reset_work))
4918                                         r = -EALREADY;
4919                         } else
4920                                 r = amdgpu_asic_reset(tmp_adev);
4921
4922                         if (r) {
4923                                 dev_err(tmp_adev->dev, "ASIC reset failed with error, %d for drm dev, %s",
4924                                          r, adev_to_drm(tmp_adev)->unique);
4925                                 break;
4926                         }
4927                 }
4928
4929                 /* For XGMI wait for all resets to complete before proceed */
4930                 if (!r) {
4931                         list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
4932                                 if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) {
4933                                         flush_work(&tmp_adev->xgmi_reset_work);
4934                                         r = tmp_adev->asic_reset_res;
4935                                         if (r)
4936                                                 break;
4937                                 }
4938                         }
4939                 }
4940         }
4941
4942         if (!r && amdgpu_ras_intr_triggered()) {
4943                 list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
4944                         if (tmp_adev->mmhub.ras && tmp_adev->mmhub.ras->ras_block.hw_ops &&
4945                             tmp_adev->mmhub.ras->ras_block.hw_ops->reset_ras_error_count)
4946                                 tmp_adev->mmhub.ras->ras_block.hw_ops->reset_ras_error_count(tmp_adev);
4947                 }
4948
4949                 amdgpu_ras_intr_cleared();
4950         }
4951
4952         /* Since the mode1 reset affects base ip blocks, the
4953          * phase1 ip blocks need to be resumed. Otherwise there
4954          * will be a BIOS signature error and the psp bootloader
4955          * can't load kdb on the next amdgpu install.
4956          */
4957         if (gpu_reset_for_dev_remove) {
4958                 list_for_each_entry(tmp_adev, device_list_handle, reset_list)
4959                         amdgpu_device_ip_resume_phase1(tmp_adev);
4960
4961                 goto end;
4962         }
4963
4964         list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
4965                 if (need_full_reset) {
4966                         /* post card */
4967                         r = amdgpu_device_asic_init(tmp_adev);
4968                         if (r) {
4969                                 dev_warn(tmp_adev->dev, "asic atom init failed!");
4970                         } else {
4971                                 dev_info(tmp_adev->dev, "GPU reset succeeded, trying to resume\n");
4972                                 r = amdgpu_amdkfd_resume_iommu(tmp_adev);
4973                                 if (r)
4974                                         goto out;
4975
4976                                 r = amdgpu_device_ip_resume_phase1(tmp_adev);
4977                                 if (r)
4978                                         goto out;
4979
4980                                 vram_lost = amdgpu_device_check_vram_lost(tmp_adev);
4981 #ifdef CONFIG_DEV_COREDUMP
4982                                 tmp_adev->reset_vram_lost = vram_lost;
4983                                 memset(&tmp_adev->reset_task_info, 0,
4984                                                 sizeof(tmp_adev->reset_task_info));
4985                                 if (reset_context->job && reset_context->job->vm)
4986                                         tmp_adev->reset_task_info =
4987                                                 reset_context->job->vm->task_info;
4988                                 amdgpu_reset_capture_coredumpm(tmp_adev);
4989 #endif
4990                                 if (vram_lost) {
4991                                         DRM_INFO("VRAM is lost due to GPU reset!\n");
4992                                         amdgpu_inc_vram_lost(tmp_adev);
4993                                 }
4994
4995                                 r = amdgpu_device_fw_loading(tmp_adev);
4996                                 if (r)
4997                                         return r;
4998
4999                                 r = amdgpu_device_ip_resume_phase2(tmp_adev);
5000                                 if (r)
5001                                         goto out;
5002
5003                                 if (vram_lost)
5004                                         amdgpu_device_fill_reset_magic(tmp_adev);
5005
5006                                 /*
5007                                  * Add this ASIC as tracked as reset was already
5008                                  * complete successfully.
5009                                  */
5010                                 amdgpu_register_gpu_instance(tmp_adev);
5011
5012                                 if (!reset_context->hive &&
5013                                     tmp_adev->gmc.xgmi.num_physical_nodes > 1)
5014                                         amdgpu_xgmi_add_device(tmp_adev);
5015
5016                                 r = amdgpu_device_ip_late_init(tmp_adev);
5017                                 if (r)
5018                                         goto out;
5019
5020                                 drm_fb_helper_set_suspend_unlocked(adev_to_drm(tmp_adev)->fb_helper, false);
5021
5022                                 /*
5023                                  * The GPU enters bad state once faulty pages
5024                                  * by ECC has reached the threshold, and ras
5025                                  * recovery is scheduled next. So add one check
5026                                  * here to break recovery if it indeed exceeds
5027                                  * bad page threshold, and remind user to
5028                                  * retire this GPU or setting one bigger
5029                                  * bad_page_threshold value to fix this once
5030                                  * probing driver again.
5031                                  */
5032                                 if (!amdgpu_ras_eeprom_check_err_threshold(tmp_adev)) {
5033                                         /* must succeed. */
5034                                         amdgpu_ras_resume(tmp_adev);
5035                                 } else {
5036                                         r = -EINVAL;
5037                                         goto out;
5038                                 }
5039
5040                                 /* Update PSP FW topology after reset */
5041                                 if (reset_context->hive &&
5042                                     tmp_adev->gmc.xgmi.num_physical_nodes > 1)
5043                                         r = amdgpu_xgmi_update_topology(
5044                                                 reset_context->hive, tmp_adev);
5045                         }
5046                 }
5047
5048 out:
5049                 if (!r) {
5050                         amdgpu_irq_gpu_reset_resume_helper(tmp_adev);
5051                         r = amdgpu_ib_ring_tests(tmp_adev);
5052                         if (r) {
5053                                 dev_err(tmp_adev->dev, "ib ring test failed (%d).\n", r);
5054                                 need_full_reset = true;
5055                                 r = -EAGAIN;
5056                                 goto end;
5057                         }
5058                 }
5059
5060                 if (!r)
5061                         r = amdgpu_device_recover_vram(tmp_adev);
5062                 else
5063                         tmp_adev->asic_reset_res = r;
5064         }
5065
5066 end:
5067         if (need_full_reset)
5068                 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5069         else
5070                 clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5071         return r;
5072 }
5073
5074 static void amdgpu_device_set_mp1_state(struct amdgpu_device *adev)
5075 {
5076
5077         switch (amdgpu_asic_reset_method(adev)) {
5078         case AMD_RESET_METHOD_MODE1:
5079                 adev->mp1_state = PP_MP1_STATE_SHUTDOWN;
5080                 break;
5081         case AMD_RESET_METHOD_MODE2:
5082                 adev->mp1_state = PP_MP1_STATE_RESET;
5083                 break;
5084         default:
5085                 adev->mp1_state = PP_MP1_STATE_NONE;
5086                 break;
5087         }
5088 }
5089
5090 static void amdgpu_device_unset_mp1_state(struct amdgpu_device *adev)
5091 {
5092         amdgpu_vf_error_trans_all(adev);
5093         adev->mp1_state = PP_MP1_STATE_NONE;
5094 }
5095
5096 static void amdgpu_device_resume_display_audio(struct amdgpu_device *adev)
5097 {
5098         struct pci_dev *p = NULL;
5099
5100         p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus),
5101                         adev->pdev->bus->number, 1);
5102         if (p) {
5103                 pm_runtime_enable(&(p->dev));
5104                 pm_runtime_resume(&(p->dev));
5105         }
5106
5107         pci_dev_put(p);
5108 }
5109
5110 static int amdgpu_device_suspend_display_audio(struct amdgpu_device *adev)
5111 {
5112         enum amd_reset_method reset_method;
5113         struct pci_dev *p = NULL;
5114         u64 expires;
5115
5116         /*
5117          * For now, only BACO and mode1 reset are confirmed
5118          * to suffer the audio issue without proper suspended.
5119          */
5120         reset_method = amdgpu_asic_reset_method(adev);
5121         if ((reset_method != AMD_RESET_METHOD_BACO) &&
5122              (reset_method != AMD_RESET_METHOD_MODE1))
5123                 return -EINVAL;
5124
5125         p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus),
5126                         adev->pdev->bus->number, 1);
5127         if (!p)
5128                 return -ENODEV;
5129
5130         expires = pm_runtime_autosuspend_expiration(&(p->dev));
5131         if (!expires)
5132                 /*
5133                  * If we cannot get the audio device autosuspend delay,
5134                  * a fixed 4S interval will be used. Considering 3S is
5135                  * the audio controller default autosuspend delay setting.
5136                  * 4S used here is guaranteed to cover that.
5137                  */
5138                 expires = ktime_get_mono_fast_ns() + NSEC_PER_SEC * 4ULL;
5139
5140         while (!pm_runtime_status_suspended(&(p->dev))) {
5141                 if (!pm_runtime_suspend(&(p->dev)))
5142                         break;
5143
5144                 if (expires < ktime_get_mono_fast_ns()) {
5145                         dev_warn(adev->dev, "failed to suspend display audio\n");
5146                         pci_dev_put(p);
5147                         /* TODO: abort the succeeding gpu reset? */
5148                         return -ETIMEDOUT;
5149                 }
5150         }
5151
5152         pm_runtime_disable(&(p->dev));
5153
5154         pci_dev_put(p);
5155         return 0;
5156 }
5157
5158 static inline void amdgpu_device_stop_pending_resets(struct amdgpu_device *adev)
5159 {
5160         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
5161
5162 #if defined(CONFIG_DEBUG_FS)
5163         if (!amdgpu_sriov_vf(adev))
5164                 cancel_work(&adev->reset_work);
5165 #endif
5166
5167         if (adev->kfd.dev)
5168                 cancel_work(&adev->kfd.reset_work);
5169
5170         if (amdgpu_sriov_vf(adev))
5171                 cancel_work(&adev->virt.flr_work);
5172
5173         if (con && adev->ras_enabled)
5174                 cancel_work(&con->recovery_work);
5175
5176 }
5177
5178 /**
5179  * amdgpu_device_gpu_recover - reset the asic and recover scheduler
5180  *
5181  * @adev: amdgpu_device pointer
5182  * @job: which job trigger hang
5183  * @reset_context: amdgpu reset context pointer
5184  *
5185  * Attempt to reset the GPU if it has hung (all asics).
5186  * Attempt to do soft-reset or full-reset and reinitialize Asic
5187  * Returns 0 for success or an error on failure.
5188  */
5189
5190 int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
5191                               struct amdgpu_job *job,
5192                               struct amdgpu_reset_context *reset_context)
5193 {
5194         struct list_head device_list, *device_list_handle =  NULL;
5195         bool job_signaled = false;
5196         struct amdgpu_hive_info *hive = NULL;
5197         struct amdgpu_device *tmp_adev = NULL;
5198         int i, r = 0;
5199         bool need_emergency_restart = false;
5200         bool audio_suspended = false;
5201         bool gpu_reset_for_dev_remove = false;
5202
5203         gpu_reset_for_dev_remove =
5204                         test_bit(AMDGPU_RESET_FOR_DEVICE_REMOVE, &reset_context->flags) &&
5205                                 test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5206
5207         /*
5208          * Special case: RAS triggered and full reset isn't supported
5209          */
5210         need_emergency_restart = amdgpu_ras_need_emergency_restart(adev);
5211
5212         /*
5213          * Flush RAM to disk so that after reboot
5214          * the user can read log and see why the system rebooted.
5215          */
5216         if (need_emergency_restart && amdgpu_ras_get_context(adev)->reboot) {
5217                 DRM_WARN("Emergency reboot.");
5218
5219                 ksys_sync_helper();
5220                 emergency_restart();
5221         }
5222
5223         dev_info(adev->dev, "GPU %s begin!\n",
5224                 need_emergency_restart ? "jobs stop":"reset");
5225
5226         if (!amdgpu_sriov_vf(adev))
5227                 hive = amdgpu_get_xgmi_hive(adev);
5228         if (hive)
5229                 mutex_lock(&hive->hive_lock);
5230
5231         reset_context->job = job;
5232         reset_context->hive = hive;
5233         /*
5234          * Build list of devices to reset.
5235          * In case we are in XGMI hive mode, resort the device list
5236          * to put adev in the 1st position.
5237          */
5238         INIT_LIST_HEAD(&device_list);
5239         if (!amdgpu_sriov_vf(adev) && (adev->gmc.xgmi.num_physical_nodes > 1)) {
5240                 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
5241                         list_add_tail(&tmp_adev->reset_list, &device_list);
5242                         if (gpu_reset_for_dev_remove && adev->shutdown)
5243                                 tmp_adev->shutdown = true;
5244                 }
5245                 if (!list_is_first(&adev->reset_list, &device_list))
5246                         list_rotate_to_front(&adev->reset_list, &device_list);
5247                 device_list_handle = &device_list;
5248         } else {
5249                 list_add_tail(&adev->reset_list, &device_list);
5250                 device_list_handle = &device_list;
5251         }
5252
5253         /* We need to lock reset domain only once both for XGMI and single device */
5254         tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device,
5255                                     reset_list);
5256         amdgpu_device_lock_reset_domain(tmp_adev->reset_domain);
5257
5258         /* block all schedulers and reset given job's ring */
5259         list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5260
5261                 amdgpu_device_set_mp1_state(tmp_adev);
5262
5263                 /*
5264                  * Try to put the audio codec into suspend state
5265                  * before gpu reset started.
5266                  *
5267                  * Due to the power domain of the graphics device
5268                  * is shared with AZ power domain. Without this,
5269                  * we may change the audio hardware from behind
5270                  * the audio driver's back. That will trigger
5271                  * some audio codec errors.
5272                  */
5273                 if (!amdgpu_device_suspend_display_audio(tmp_adev))
5274                         audio_suspended = true;
5275
5276                 amdgpu_ras_set_error_query_ready(tmp_adev, false);
5277
5278                 cancel_delayed_work_sync(&tmp_adev->delayed_init_work);
5279
5280                 if (!amdgpu_sriov_vf(tmp_adev))
5281                         amdgpu_amdkfd_pre_reset(tmp_adev);
5282
5283                 /*
5284                  * Mark these ASICs to be reseted as untracked first
5285                  * And add them back after reset completed
5286                  */
5287                 amdgpu_unregister_gpu_instance(tmp_adev);
5288
5289                 drm_fb_helper_set_suspend_unlocked(adev_to_drm(tmp_adev)->fb_helper, true);
5290
5291                 /* disable ras on ALL IPs */
5292                 if (!need_emergency_restart &&
5293                       amdgpu_device_ip_need_full_reset(tmp_adev))
5294                         amdgpu_ras_suspend(tmp_adev);
5295
5296                 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5297                         struct amdgpu_ring *ring = tmp_adev->rings[i];
5298
5299                         if (!ring || !ring->sched.thread)
5300                                 continue;
5301
5302                         drm_sched_stop(&ring->sched, job ? &job->base : NULL);
5303
5304                         if (need_emergency_restart)
5305                                 amdgpu_job_stop_all_jobs_on_sched(&ring->sched);
5306                 }
5307                 atomic_inc(&tmp_adev->gpu_reset_counter);
5308         }
5309
5310         if (need_emergency_restart)
5311                 goto skip_sched_resume;
5312
5313         /*
5314          * Must check guilty signal here since after this point all old
5315          * HW fences are force signaled.
5316          *
5317          * job->base holds a reference to parent fence
5318          */
5319         if (job && dma_fence_is_signaled(&job->hw_fence)) {
5320                 job_signaled = true;
5321                 dev_info(adev->dev, "Guilty job already signaled, skipping HW reset");
5322                 goto skip_hw_reset;
5323         }
5324
5325 retry:  /* Rest of adevs pre asic reset from XGMI hive. */
5326         list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5327                 if (gpu_reset_for_dev_remove) {
5328                         /* Workaroud for ASICs need to disable SMC first */
5329                         amdgpu_device_smu_fini_early(tmp_adev);
5330                 }
5331                 r = amdgpu_device_pre_asic_reset(tmp_adev, reset_context);
5332                 /*TODO Should we stop ?*/
5333                 if (r) {
5334                         dev_err(tmp_adev->dev, "GPU pre asic reset failed with err, %d for drm dev, %s ",
5335                                   r, adev_to_drm(tmp_adev)->unique);
5336                         tmp_adev->asic_reset_res = r;
5337                 }
5338
5339                 /*
5340                  * Drop all pending non scheduler resets. Scheduler resets
5341                  * were already dropped during drm_sched_stop
5342                  */
5343                 amdgpu_device_stop_pending_resets(tmp_adev);
5344         }
5345
5346         /* Actual ASIC resets if needed.*/
5347         /* Host driver will handle XGMI hive reset for SRIOV */
5348         if (amdgpu_sriov_vf(adev)) {
5349                 r = amdgpu_device_reset_sriov(adev, job ? false : true);
5350                 if (r)
5351                         adev->asic_reset_res = r;
5352
5353                 /* Aldebaran and gfx_11_0_3 support ras in SRIOV, so need resume ras during reset */
5354                 if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 2) ||
5355                     adev->ip_versions[GC_HWIP][0] == IP_VERSION(11, 0, 3))
5356                         amdgpu_ras_resume(adev);
5357         } else {
5358                 r = amdgpu_do_asic_reset(device_list_handle, reset_context);
5359                 if (r && r == -EAGAIN)
5360                         goto retry;
5361
5362                 if (!r && gpu_reset_for_dev_remove)
5363                         goto recover_end;
5364         }
5365
5366 skip_hw_reset:
5367
5368         /* Post ASIC reset for all devs .*/
5369         list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5370
5371                 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5372                         struct amdgpu_ring *ring = tmp_adev->rings[i];
5373
5374                         if (!ring || !ring->sched.thread)
5375                                 continue;
5376
5377                         drm_sched_start(&ring->sched, true);
5378                 }
5379
5380                 if (adev->enable_mes && adev->ip_versions[GC_HWIP][0] != IP_VERSION(11, 0, 3))
5381                         amdgpu_mes_self_test(tmp_adev);
5382
5383                 if (!drm_drv_uses_atomic_modeset(adev_to_drm(tmp_adev)) && !job_signaled) {
5384                         drm_helper_resume_force_mode(adev_to_drm(tmp_adev));
5385                 }
5386
5387                 if (tmp_adev->asic_reset_res)
5388                         r = tmp_adev->asic_reset_res;
5389
5390                 tmp_adev->asic_reset_res = 0;
5391
5392                 if (r) {
5393                         /* bad news, how to tell it to userspace ? */
5394                         dev_info(tmp_adev->dev, "GPU reset(%d) failed\n", atomic_read(&tmp_adev->gpu_reset_counter));
5395                         amdgpu_vf_error_put(tmp_adev, AMDGIM_ERROR_VF_GPU_RESET_FAIL, 0, r);
5396                 } else {
5397                         dev_info(tmp_adev->dev, "GPU reset(%d) succeeded!\n", atomic_read(&tmp_adev->gpu_reset_counter));
5398                         if (amdgpu_acpi_smart_shift_update(adev_to_drm(tmp_adev), AMDGPU_SS_DEV_D0))
5399                                 DRM_WARN("smart shift update failed\n");
5400                 }
5401         }
5402
5403 skip_sched_resume:
5404         list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5405                 /* unlock kfd: SRIOV would do it separately */
5406                 if (!need_emergency_restart && !amdgpu_sriov_vf(tmp_adev))
5407                         amdgpu_amdkfd_post_reset(tmp_adev);
5408
5409                 /* kfd_post_reset will do nothing if kfd device is not initialized,
5410                  * need to bring up kfd here if it's not be initialized before
5411                  */
5412                 if (!adev->kfd.init_complete)
5413                         amdgpu_amdkfd_device_init(adev);
5414
5415                 if (audio_suspended)
5416                         amdgpu_device_resume_display_audio(tmp_adev);
5417
5418                 amdgpu_device_unset_mp1_state(tmp_adev);
5419
5420                 amdgpu_ras_set_error_query_ready(tmp_adev, true);
5421         }
5422
5423 recover_end:
5424         tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device,
5425                                             reset_list);
5426         amdgpu_device_unlock_reset_domain(tmp_adev->reset_domain);
5427
5428         if (hive) {
5429                 mutex_unlock(&hive->hive_lock);
5430                 amdgpu_put_xgmi_hive(hive);
5431         }
5432
5433         if (r)
5434                 dev_info(adev->dev, "GPU reset end with ret = %d\n", r);
5435
5436         atomic_set(&adev->reset_domain->reset_res, r);
5437         return r;
5438 }
5439
5440 /**
5441  * amdgpu_device_get_pcie_info - fence pcie info about the PCIE slot
5442  *
5443  * @adev: amdgpu_device pointer
5444  *
5445  * Fetchs and stores in the driver the PCIE capabilities (gen speed
5446  * and lanes) of the slot the device is in. Handles APUs and
5447  * virtualized environments where PCIE config space may not be available.
5448  */
5449 static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev)
5450 {
5451         struct pci_dev *pdev;
5452         enum pci_bus_speed speed_cap, platform_speed_cap;
5453         enum pcie_link_width platform_link_width;
5454
5455         if (amdgpu_pcie_gen_cap)
5456                 adev->pm.pcie_gen_mask = amdgpu_pcie_gen_cap;
5457
5458         if (amdgpu_pcie_lane_cap)
5459                 adev->pm.pcie_mlw_mask = amdgpu_pcie_lane_cap;
5460
5461         /* covers APUs as well */
5462         if (pci_is_root_bus(adev->pdev->bus)) {
5463                 if (adev->pm.pcie_gen_mask == 0)
5464                         adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK;
5465                 if (adev->pm.pcie_mlw_mask == 0)
5466                         adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK;
5467                 return;
5468         }
5469
5470         if (adev->pm.pcie_gen_mask && adev->pm.pcie_mlw_mask)
5471                 return;
5472
5473         pcie_bandwidth_available(adev->pdev, NULL,
5474                                  &platform_speed_cap, &platform_link_width);
5475
5476         if (adev->pm.pcie_gen_mask == 0) {
5477                 /* asic caps */
5478                 pdev = adev->pdev;
5479                 speed_cap = pcie_get_speed_cap(pdev);
5480                 if (speed_cap == PCI_SPEED_UNKNOWN) {
5481                         adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5482                                                   CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5483                                                   CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
5484                 } else {
5485                         if (speed_cap == PCIE_SPEED_32_0GT)
5486                                 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5487                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5488                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5489                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4 |
5490                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN5);
5491                         else if (speed_cap == PCIE_SPEED_16_0GT)
5492                                 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5493                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5494                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5495                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4);
5496                         else if (speed_cap == PCIE_SPEED_8_0GT)
5497                                 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5498                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5499                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
5500                         else if (speed_cap == PCIE_SPEED_5_0GT)
5501                                 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5502                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2);
5503                         else
5504                                 adev->pm.pcie_gen_mask |= CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1;
5505                 }
5506                 /* platform caps */
5507                 if (platform_speed_cap == PCI_SPEED_UNKNOWN) {
5508                         adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5509                                                    CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2);
5510                 } else {
5511                         if (platform_speed_cap == PCIE_SPEED_32_0GT)
5512                                 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5513                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5514                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5515                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4 |
5516                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN5);
5517                         else if (platform_speed_cap == PCIE_SPEED_16_0GT)
5518                                 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5519                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5520                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5521                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4);
5522                         else if (platform_speed_cap == PCIE_SPEED_8_0GT)
5523                                 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5524                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5525                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3);
5526                         else if (platform_speed_cap == PCIE_SPEED_5_0GT)
5527                                 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5528                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2);
5529                         else
5530                                 adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1;
5531
5532                 }
5533         }
5534         if (adev->pm.pcie_mlw_mask == 0) {
5535                 if (platform_link_width == PCIE_LNK_WIDTH_UNKNOWN) {
5536                         adev->pm.pcie_mlw_mask |= AMDGPU_DEFAULT_PCIE_MLW_MASK;
5537                 } else {
5538                         switch (platform_link_width) {
5539                         case PCIE_LNK_X32:
5540                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X32 |
5541                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
5542                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
5543                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5544                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5545                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5546                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5547                                 break;
5548                         case PCIE_LNK_X16:
5549                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
5550                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
5551                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5552                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5553                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5554                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5555                                 break;
5556                         case PCIE_LNK_X12:
5557                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
5558                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5559                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5560                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5561                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5562                                 break;
5563                         case PCIE_LNK_X8:
5564                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5565                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5566                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5567                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5568                                 break;
5569                         case PCIE_LNK_X4:
5570                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5571                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5572                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5573                                 break;
5574                         case PCIE_LNK_X2:
5575                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5576                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5577                                 break;
5578                         case PCIE_LNK_X1:
5579                                 adev->pm.pcie_mlw_mask = CAIL_PCIE_LINK_WIDTH_SUPPORT_X1;
5580                                 break;
5581                         default:
5582                                 break;
5583                         }
5584                 }
5585         }
5586 }
5587
5588 /**
5589  * amdgpu_device_is_peer_accessible - Check peer access through PCIe BAR
5590  *
5591  * @adev: amdgpu_device pointer
5592  * @peer_adev: amdgpu_device pointer for peer device trying to access @adev
5593  *
5594  * Return true if @peer_adev can access (DMA) @adev through the PCIe
5595  * BAR, i.e. @adev is "large BAR" and the BAR matches the DMA mask of
5596  * @peer_adev.
5597  */
5598 bool amdgpu_device_is_peer_accessible(struct amdgpu_device *adev,
5599                                       struct amdgpu_device *peer_adev)
5600 {
5601 #ifdef CONFIG_HSA_AMD_P2P
5602         uint64_t address_mask = peer_adev->dev->dma_mask ?
5603                 ~*peer_adev->dev->dma_mask : ~((1ULL << 32) - 1);
5604         resource_size_t aper_limit =
5605                 adev->gmc.aper_base + adev->gmc.aper_size - 1;
5606         bool p2p_access =
5607                 !adev->gmc.xgmi.connected_to_cpu &&
5608                 !(pci_p2pdma_distance(adev->pdev, peer_adev->dev, false) < 0);
5609
5610         return pcie_p2p && p2p_access && (adev->gmc.visible_vram_size &&
5611                 adev->gmc.real_vram_size == adev->gmc.visible_vram_size &&
5612                 !(adev->gmc.aper_base & address_mask ||
5613                   aper_limit & address_mask));
5614 #else
5615         return false;
5616 #endif
5617 }
5618
5619 int amdgpu_device_baco_enter(struct drm_device *dev)
5620 {
5621         struct amdgpu_device *adev = drm_to_adev(dev);
5622         struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
5623
5624         if (!amdgpu_device_supports_baco(dev))
5625                 return -ENOTSUPP;
5626
5627         if (ras && adev->ras_enabled &&
5628             adev->nbio.funcs->enable_doorbell_interrupt)
5629                 adev->nbio.funcs->enable_doorbell_interrupt(adev, false);
5630
5631         return amdgpu_dpm_baco_enter(adev);
5632 }
5633
5634 int amdgpu_device_baco_exit(struct drm_device *dev)
5635 {
5636         struct amdgpu_device *adev = drm_to_adev(dev);
5637         struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
5638         int ret = 0;
5639
5640         if (!amdgpu_device_supports_baco(dev))
5641                 return -ENOTSUPP;
5642
5643         ret = amdgpu_dpm_baco_exit(adev);
5644         if (ret)
5645                 return ret;
5646
5647         if (ras && adev->ras_enabled &&
5648             adev->nbio.funcs->enable_doorbell_interrupt)
5649                 adev->nbio.funcs->enable_doorbell_interrupt(adev, true);
5650
5651         if (amdgpu_passthrough(adev) &&
5652             adev->nbio.funcs->clear_doorbell_interrupt)
5653                 adev->nbio.funcs->clear_doorbell_interrupt(adev);
5654
5655         return 0;
5656 }
5657
5658 /**
5659  * amdgpu_pci_error_detected - Called when a PCI error is detected.
5660  * @pdev: PCI device struct
5661  * @state: PCI channel state
5662  *
5663  * Description: Called when a PCI error is detected.
5664  *
5665  * Return: PCI_ERS_RESULT_NEED_RESET or PCI_ERS_RESULT_DISCONNECT.
5666  */
5667 pci_ers_result_t amdgpu_pci_error_detected(struct pci_dev *pdev, pci_channel_state_t state)
5668 {
5669         struct drm_device *dev = pci_get_drvdata(pdev);
5670         struct amdgpu_device *adev = drm_to_adev(dev);
5671         int i;
5672
5673         DRM_INFO("PCI error: detected callback, state(%d)!!\n", state);
5674
5675         if (adev->gmc.xgmi.num_physical_nodes > 1) {
5676                 DRM_WARN("No support for XGMI hive yet...");
5677                 return PCI_ERS_RESULT_DISCONNECT;
5678         }
5679
5680         adev->pci_channel_state = state;
5681
5682         switch (state) {
5683         case pci_channel_io_normal:
5684                 return PCI_ERS_RESULT_CAN_RECOVER;
5685         /* Fatal error, prepare for slot reset */
5686         case pci_channel_io_frozen:
5687                 /*
5688                  * Locking adev->reset_domain->sem will prevent any external access
5689                  * to GPU during PCI error recovery
5690                  */
5691                 amdgpu_device_lock_reset_domain(adev->reset_domain);
5692                 amdgpu_device_set_mp1_state(adev);
5693
5694                 /*
5695                  * Block any work scheduling as we do for regular GPU reset
5696                  * for the duration of the recovery
5697                  */
5698                 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5699                         struct amdgpu_ring *ring = adev->rings[i];
5700
5701                         if (!ring || !ring->sched.thread)
5702                                 continue;
5703
5704                         drm_sched_stop(&ring->sched, NULL);
5705                 }
5706                 atomic_inc(&adev->gpu_reset_counter);
5707                 return PCI_ERS_RESULT_NEED_RESET;
5708         case pci_channel_io_perm_failure:
5709                 /* Permanent error, prepare for device removal */
5710                 return PCI_ERS_RESULT_DISCONNECT;
5711         }
5712
5713         return PCI_ERS_RESULT_NEED_RESET;
5714 }
5715
5716 /**
5717  * amdgpu_pci_mmio_enabled - Enable MMIO and dump debug registers
5718  * @pdev: pointer to PCI device
5719  */
5720 pci_ers_result_t amdgpu_pci_mmio_enabled(struct pci_dev *pdev)
5721 {
5722
5723         DRM_INFO("PCI error: mmio enabled callback!!\n");
5724
5725         /* TODO - dump whatever for debugging purposes */
5726
5727         /* This called only if amdgpu_pci_error_detected returns
5728          * PCI_ERS_RESULT_CAN_RECOVER. Read/write to the device still
5729          * works, no need to reset slot.
5730          */
5731
5732         return PCI_ERS_RESULT_RECOVERED;
5733 }
5734
5735 /**
5736  * amdgpu_pci_slot_reset - Called when PCI slot has been reset.
5737  * @pdev: PCI device struct
5738  *
5739  * Description: This routine is called by the pci error recovery
5740  * code after the PCI slot has been reset, just before we
5741  * should resume normal operations.
5742  */
5743 pci_ers_result_t amdgpu_pci_slot_reset(struct pci_dev *pdev)
5744 {
5745         struct drm_device *dev = pci_get_drvdata(pdev);
5746         struct amdgpu_device *adev = drm_to_adev(dev);
5747         int r, i;
5748         struct amdgpu_reset_context reset_context;
5749         u32 memsize;
5750         struct list_head device_list;
5751
5752         DRM_INFO("PCI error: slot reset callback!!\n");
5753
5754         memset(&reset_context, 0, sizeof(reset_context));
5755
5756         INIT_LIST_HEAD(&device_list);
5757         list_add_tail(&adev->reset_list, &device_list);
5758
5759         /* wait for asic to come out of reset */
5760         msleep(500);
5761
5762         /* Restore PCI confspace */
5763         amdgpu_device_load_pci_state(pdev);
5764
5765         /* confirm  ASIC came out of reset */
5766         for (i = 0; i < adev->usec_timeout; i++) {
5767                 memsize = amdgpu_asic_get_config_memsize(adev);
5768
5769                 if (memsize != 0xffffffff)
5770                         break;
5771                 udelay(1);
5772         }
5773         if (memsize == 0xffffffff) {
5774                 r = -ETIME;
5775                 goto out;
5776         }
5777
5778         reset_context.method = AMD_RESET_METHOD_NONE;
5779         reset_context.reset_req_dev = adev;
5780         set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
5781         set_bit(AMDGPU_SKIP_HW_RESET, &reset_context.flags);
5782
5783         adev->no_hw_access = true;
5784         r = amdgpu_device_pre_asic_reset(adev, &reset_context);
5785         adev->no_hw_access = false;
5786         if (r)
5787                 goto out;
5788
5789         r = amdgpu_do_asic_reset(&device_list, &reset_context);
5790
5791 out:
5792         if (!r) {
5793                 if (amdgpu_device_cache_pci_state(adev->pdev))
5794                         pci_restore_state(adev->pdev);
5795
5796                 DRM_INFO("PCIe error recovery succeeded\n");
5797         } else {
5798                 DRM_ERROR("PCIe error recovery failed, err:%d", r);
5799                 amdgpu_device_unset_mp1_state(adev);
5800                 amdgpu_device_unlock_reset_domain(adev->reset_domain);
5801         }
5802
5803         return r ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_RECOVERED;
5804 }
5805
5806 /**
5807  * amdgpu_pci_resume() - resume normal ops after PCI reset
5808  * @pdev: pointer to PCI device
5809  *
5810  * Called when the error recovery driver tells us that its
5811  * OK to resume normal operation.
5812  */
5813 void amdgpu_pci_resume(struct pci_dev *pdev)
5814 {
5815         struct drm_device *dev = pci_get_drvdata(pdev);
5816         struct amdgpu_device *adev = drm_to_adev(dev);
5817         int i;
5818
5819
5820         DRM_INFO("PCI error: resume callback!!\n");
5821
5822         /* Only continue execution for the case of pci_channel_io_frozen */
5823         if (adev->pci_channel_state != pci_channel_io_frozen)
5824                 return;
5825
5826         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5827                 struct amdgpu_ring *ring = adev->rings[i];
5828
5829                 if (!ring || !ring->sched.thread)
5830                         continue;
5831
5832                 drm_sched_start(&ring->sched, true);
5833         }
5834
5835         amdgpu_device_unset_mp1_state(adev);
5836         amdgpu_device_unlock_reset_domain(adev->reset_domain);
5837 }
5838
5839 bool amdgpu_device_cache_pci_state(struct pci_dev *pdev)
5840 {
5841         struct drm_device *dev = pci_get_drvdata(pdev);
5842         struct amdgpu_device *adev = drm_to_adev(dev);
5843         int r;
5844
5845         r = pci_save_state(pdev);
5846         if (!r) {
5847                 kfree(adev->pci_state);
5848
5849                 adev->pci_state = pci_store_saved_state(pdev);
5850
5851                 if (!adev->pci_state) {
5852                         DRM_ERROR("Failed to store PCI saved state");
5853                         return false;
5854                 }
5855         } else {
5856                 DRM_WARN("Failed to save PCI state, err:%d\n", r);
5857                 return false;
5858         }
5859
5860         return true;
5861 }
5862
5863 bool amdgpu_device_load_pci_state(struct pci_dev *pdev)
5864 {
5865         struct drm_device *dev = pci_get_drvdata(pdev);
5866         struct amdgpu_device *adev = drm_to_adev(dev);
5867         int r;
5868
5869         if (!adev->pci_state)
5870                 return false;
5871
5872         r = pci_load_saved_state(pdev, adev->pci_state);
5873
5874         if (!r) {
5875                 pci_restore_state(pdev);
5876         } else {
5877                 DRM_WARN("Failed to load PCI state, err:%d\n", r);
5878                 return false;
5879         }
5880
5881         return true;
5882 }
5883
5884 void amdgpu_device_flush_hdp(struct amdgpu_device *adev,
5885                 struct amdgpu_ring *ring)
5886 {
5887 #ifdef CONFIG_X86_64
5888         if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev))
5889                 return;
5890 #endif
5891         if (adev->gmc.xgmi.connected_to_cpu)
5892                 return;
5893
5894         if (ring && ring->funcs->emit_hdp_flush)
5895                 amdgpu_ring_emit_hdp_flush(ring);
5896         else
5897                 amdgpu_asic_flush_hdp(adev, ring);
5898 }
5899
5900 void amdgpu_device_invalidate_hdp(struct amdgpu_device *adev,
5901                 struct amdgpu_ring *ring)
5902 {
5903 #ifdef CONFIG_X86_64
5904         if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev))
5905                 return;
5906 #endif
5907         if (adev->gmc.xgmi.connected_to_cpu)
5908                 return;
5909
5910         amdgpu_asic_invalidate_hdp(adev, ring);
5911 }
5912
5913 int amdgpu_in_reset(struct amdgpu_device *adev)
5914 {
5915         return atomic_read(&adev->reset_domain->in_gpu_reset);
5916 }
5917
5918 /**
5919  * amdgpu_device_halt() - bring hardware to some kind of halt state
5920  *
5921  * @adev: amdgpu_device pointer
5922  *
5923  * Bring hardware to some kind of halt state so that no one can touch it
5924  * any more. It will help to maintain error context when error occurred.
5925  * Compare to a simple hang, the system will keep stable at least for SSH
5926  * access. Then it should be trivial to inspect the hardware state and
5927  * see what's going on. Implemented as following:
5928  *
5929  * 1. drm_dev_unplug() makes device inaccessible to user space(IOCTLs, etc),
5930  *    clears all CPU mappings to device, disallows remappings through page faults
5931  * 2. amdgpu_irq_disable_all() disables all interrupts
5932  * 3. amdgpu_fence_driver_hw_fini() signals all HW fences
5933  * 4. set adev->no_hw_access to avoid potential crashes after setp 5
5934  * 5. amdgpu_device_unmap_mmio() clears all MMIO mappings
5935  * 6. pci_disable_device() and pci_wait_for_pending_transaction()
5936  *    flush any in flight DMA operations
5937  */
5938 void amdgpu_device_halt(struct amdgpu_device *adev)
5939 {
5940         struct pci_dev *pdev = adev->pdev;
5941         struct drm_device *ddev = adev_to_drm(adev);
5942
5943         drm_dev_unplug(ddev);
5944
5945         amdgpu_irq_disable_all(adev);
5946
5947         amdgpu_fence_driver_hw_fini(adev);
5948
5949         adev->no_hw_access = true;
5950
5951         amdgpu_device_unmap_mmio(adev);
5952
5953         pci_disable_device(pdev);
5954         pci_wait_for_pending_transaction(pdev);
5955 }
5956
5957 u32 amdgpu_device_pcie_port_rreg(struct amdgpu_device *adev,
5958                                 u32 reg)
5959 {
5960         unsigned long flags, address, data;
5961         u32 r;
5962
5963         address = adev->nbio.funcs->get_pcie_port_index_offset(adev);
5964         data = adev->nbio.funcs->get_pcie_port_data_offset(adev);
5965
5966         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
5967         WREG32(address, reg * 4);
5968         (void)RREG32(address);
5969         r = RREG32(data);
5970         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
5971         return r;
5972 }
5973
5974 void amdgpu_device_pcie_port_wreg(struct amdgpu_device *adev,
5975                                 u32 reg, u32 v)
5976 {
5977         unsigned long flags, address, data;
5978
5979         address = adev->nbio.funcs->get_pcie_port_index_offset(adev);
5980         data = adev->nbio.funcs->get_pcie_port_data_offset(adev);
5981
5982         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
5983         WREG32(address, reg * 4);
5984         (void)RREG32(address);
5985         WREG32(data, v);
5986         (void)RREG32(data);
5987         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
5988 }
5989
5990 /**
5991  * amdgpu_device_switch_gang - switch to a new gang
5992  * @adev: amdgpu_device pointer
5993  * @gang: the gang to switch to
5994  *
5995  * Try to switch to a new gang.
5996  * Returns: NULL if we switched to the new gang or a reference to the current
5997  * gang leader.
5998  */
5999 struct dma_fence *amdgpu_device_switch_gang(struct amdgpu_device *adev,
6000                                             struct dma_fence *gang)
6001 {
6002         struct dma_fence *old = NULL;
6003
6004         do {
6005                 dma_fence_put(old);
6006                 rcu_read_lock();
6007                 old = dma_fence_get_rcu_safe(&adev->gang_submit);
6008                 rcu_read_unlock();
6009
6010                 if (old == gang)
6011                         break;
6012
6013                 if (!dma_fence_is_signaled(old))
6014                         return old;
6015
6016         } while (cmpxchg((struct dma_fence __force **)&adev->gang_submit,
6017                          old, gang) != old);
6018
6019         dma_fence_put(old);
6020         return NULL;
6021 }
6022
6023 bool amdgpu_device_has_display_hardware(struct amdgpu_device *adev)
6024 {
6025         switch (adev->asic_type) {
6026 #ifdef CONFIG_DRM_AMDGPU_SI
6027         case CHIP_HAINAN:
6028 #endif
6029         case CHIP_TOPAZ:
6030                 /* chips with no display hardware */
6031                 return false;
6032 #ifdef CONFIG_DRM_AMDGPU_SI
6033         case CHIP_TAHITI:
6034         case CHIP_PITCAIRN:
6035         case CHIP_VERDE:
6036         case CHIP_OLAND:
6037 #endif
6038 #ifdef CONFIG_DRM_AMDGPU_CIK
6039         case CHIP_BONAIRE:
6040         case CHIP_HAWAII:
6041         case CHIP_KAVERI:
6042         case CHIP_KABINI:
6043         case CHIP_MULLINS:
6044 #endif
6045         case CHIP_TONGA:
6046         case CHIP_FIJI:
6047         case CHIP_POLARIS10:
6048         case CHIP_POLARIS11:
6049         case CHIP_POLARIS12:
6050         case CHIP_VEGAM:
6051         case CHIP_CARRIZO:
6052         case CHIP_STONEY:
6053                 /* chips with display hardware */
6054                 return true;
6055         default:
6056                 /* IP discovery */
6057                 if (!adev->ip_versions[DCE_HWIP][0] ||
6058                     (adev->harvest_ip_mask & AMD_HARVEST_IP_DMU_MASK))
6059                         return false;
6060                 return true;
6061         }
6062 }