drivers/gpu/drm/amd/amdgpu/amdgpu_device.c

   1 /*
   2  * Copyright 2008 Advanced Micro Devices, Inc.
   3  * Copyright 2008 Red Hat Inc.
   4  * Copyright 2009 Jerome Glisse.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22  * OTHER DEALINGS IN THE SOFTWARE.
  23  *
  24  * Authors: Dave Airlie
  25  *          Alex Deucher
  26  *          Jerome Glisse
  27  */
  28 #include <linux/power_supply.h>
  29 #include <linux/kthread.h>
  30 #include <linux/module.h>
  31 #include <linux/console.h>
  32 #include <linux/slab.h>
  33 #include <linux/iommu.h>
  34 #include <linux/pci.h>
  35 #include <linux/pci-p2pdma.h>
  36 #include <linux/apple-gmux.h>
  37
  38 #include <drm/drm_aperture.h>
  39 #include <drm/drm_atomic_helper.h>
  40 #include <drm/drm_crtc_helper.h>
  41 #include <drm/drm_fb_helper.h>
  42 #include <drm/drm_probe_helper.h>
  43 #include <drm/amdgpu_drm.h>
  44 #include <linux/device.h>
  45 #include <linux/vgaarb.h>
  46 #include <linux/vga_switcheroo.h>
  47 #include <linux/efi.h>
  48 #include "amdgpu.h"
  49 #include "amdgpu_trace.h"
  50 #include "amdgpu_i2c.h"
  51 #include "atom.h"
  52 #include "amdgpu_atombios.h"
  53 #include "amdgpu_atomfirmware.h"
  54 #include "amd_pcie.h"
  55 #ifdef CONFIG_DRM_AMDGPU_SI
  56 #include "si.h"
  57 #endif
  58 #ifdef CONFIG_DRM_AMDGPU_CIK
  59 #include "cik.h"
  60 #endif
  61 #include "vi.h"
  62 #include "soc15.h"
  63 #include "nv.h"
  64 #include "bif/bif_4_1_d.h"
  65 #include <linux/firmware.h>
  66 #include "amdgpu_vf_error.h"
  67
  68 #include "amdgpu_amdkfd.h"
  69 #include "amdgpu_pm.h"
  70
  71 #include "amdgpu_xgmi.h"
  72 #include "amdgpu_ras.h"
  73 #include "amdgpu_pmu.h"
  74 #include "amdgpu_fru_eeprom.h"
  75 #include "amdgpu_reset.h"
  76
  77 #include <linux/suspend.h>
  78 #include <drm/task_barrier.h>
  79 #include <linux/pm_runtime.h>
  80
  81 #include <drm/drm_drv.h>
  82
  83 #if IS_ENABLED(CONFIG_X86)
  84 #include <asm/intel-family.h>
  85 #endif
  86
  87 MODULE_FIRMWARE("amdgpu/vega10_gpu_info.bin");
  88 MODULE_FIRMWARE("amdgpu/vega12_gpu_info.bin");
  89 MODULE_FIRMWARE("amdgpu/raven_gpu_info.bin");
  90 MODULE_FIRMWARE("amdgpu/picasso_gpu_info.bin");
  91 MODULE_FIRMWARE("amdgpu/raven2_gpu_info.bin");
  92 MODULE_FIRMWARE("amdgpu/arcturus_gpu_info.bin");
  93 MODULE_FIRMWARE("amdgpu/navi12_gpu_info.bin");
  94
  95 #define AMDGPU_RESUME_MS                2000
  96 #define AMDGPU_MAX_RETRY_LIMIT          2
  97 #define AMDGPU_RETRY_SRIOV_RESET(r) ((r) == -EBUSY || (r) == -ETIMEDOUT || (r) == -EINVAL)
  98
  99 static const struct drm_driver amdgpu_kms_driver;
 100
 101 const char *amdgpu_asic_name[] = {
 102         "TAHITI",
 103         "PITCAIRN",
 104         "VERDE",
 105         "OLAND",
 106         "HAINAN",
 107         "BONAIRE",
 108         "KAVERI",
 109         "KABINI",
 110         "HAWAII",
 111         "MULLINS",
 112         "TOPAZ",
 113         "TONGA",
 114         "FIJI",
 115         "CARRIZO",
 116         "STONEY",
 117         "POLARIS10",
 118         "POLARIS11",
 119         "POLARIS12",
 120         "VEGAM",
 121         "VEGA10",
 122         "VEGA12",
 123         "VEGA20",
 124         "RAVEN",
 125         "ARCTURUS",
 126         "RENOIR",
 127         "ALDEBARAN",
 128         "NAVI10",
 129         "CYAN_SKILLFISH",
 130         "NAVI14",
 131         "NAVI12",
 132         "SIENNA_CICHLID",
 133         "NAVY_FLOUNDER",
 134         "VANGOGH",
 135         "DIMGREY_CAVEFISH",
 136         "BEIGE_GOBY",
 137         "YELLOW_CARP",
 138         "IP DISCOVERY",
 139         "LAST",
 140 };
 141
 142 /**
 143  * DOC: pcie_replay_count
 144  *
 145  * The amdgpu driver provides a sysfs API for reporting the total number
 146  * of PCIe replays (NAKs)
 147  * The file pcie_replay_count is used for this and returns the total
 148  * number of replays as a sum of the NAKs generated and NAKs received
 149  */
 150
 151 static ssize_t amdgpu_device_get_pcie_replay_count(struct device *dev,
 152                 struct device_attribute *attr, char *buf)
 153 {
 154         struct drm_device *ddev = dev_get_drvdata(dev);
 155         struct amdgpu_device *adev = drm_to_adev(ddev);
 156         uint64_t cnt = amdgpu_asic_get_pcie_replay_count(adev);
 157
 158         return sysfs_emit(buf, "%llu\n", cnt);
 159 }
 160
 161 static DEVICE_ATTR(pcie_replay_count, 0444,
 162                 amdgpu_device_get_pcie_replay_count, NULL);
 163
 164 /**
 165  * DOC: board_info
 166  *
 167  * The amdgpu driver provides a sysfs API for giving board related information.
 168  * It provides the form factor information in the format
 169  *
 170  *   type : form factor
 171  *
 172  * Possible form factor values
 173  *
 174  * - "cem"              - PCIE CEM card
 175  * - "oam"              - Open Compute Accelerator Module
 176  * - "unknown"  - Not known
 177  *
 178  */
 179
 180 static ssize_t amdgpu_device_get_board_info(struct device *dev,
 181                                             struct device_attribute *attr,
 182                                             char *buf)
 183 {
 184         struct drm_device *ddev = dev_get_drvdata(dev);
 185         struct amdgpu_device *adev = drm_to_adev(ddev);
 186         enum amdgpu_pkg_type pkg_type = AMDGPU_PKG_TYPE_CEM;
 187         const char *pkg;
 188
 189         if (adev->smuio.funcs && adev->smuio.funcs->get_pkg_type)
 190                 pkg_type = adev->smuio.funcs->get_pkg_type(adev);
 191
 192         switch (pkg_type) {
 193         case AMDGPU_PKG_TYPE_CEM:
 194                 pkg = "cem";
 195                 break;
 196         case AMDGPU_PKG_TYPE_OAM:
 197                 pkg = "oam";
 198                 break;
 199         default:
 200                 pkg = "unknown";
 201                 break;
 202         }
 203
 204         return sysfs_emit(buf, "%s : %s\n", "type", pkg);
 205 }
 206
 207 static DEVICE_ATTR(board_info, 0444, amdgpu_device_get_board_info, NULL);
 208
 209 static struct attribute *amdgpu_board_attrs[] = {
 210         &dev_attr_board_info.attr,
 211         NULL,
 212 };
 213
 214 static umode_t amdgpu_board_attrs_is_visible(struct kobject *kobj,
 215                                              struct attribute *attr, int n)
 216 {
 217         struct device *dev = kobj_to_dev(kobj);
 218         struct drm_device *ddev = dev_get_drvdata(dev);
 219         struct amdgpu_device *adev = drm_to_adev(ddev);
 220
 221         if (adev->flags & AMD_IS_APU)
 222                 return 0;
 223
 224         return attr->mode;
 225 }
 226
 227 static const struct attribute_group amdgpu_board_attrs_group = {
 228         .attrs = amdgpu_board_attrs,
 229         .is_visible = amdgpu_board_attrs_is_visible
 230 };
 231
 232 static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev);
 233
 234
 235 /**
 236  * amdgpu_device_supports_px - Is the device a dGPU with ATPX power control
 237  *
 238  * @dev: drm_device pointer
 239  *
 240  * Returns true if the device is a dGPU with ATPX power control,
 241  * otherwise return false.
 242  */
 243 bool amdgpu_device_supports_px(struct drm_device *dev)
 244 {
 245         struct amdgpu_device *adev = drm_to_adev(dev);
 246
 247         if ((adev->flags & AMD_IS_PX) && !amdgpu_is_atpx_hybrid())
 248                 return true;
 249         return false;
 250 }
 251
 252 /**
 253  * amdgpu_device_supports_boco - Is the device a dGPU with ACPI power resources
 254  *
 255  * @dev: drm_device pointer
 256  *
 257  * Returns true if the device is a dGPU with ACPI power control,
 258  * otherwise return false.
 259  */
 260 bool amdgpu_device_supports_boco(struct drm_device *dev)
 261 {
 262         struct amdgpu_device *adev = drm_to_adev(dev);
 263
 264         if (adev->has_pr3 ||
 265             ((adev->flags & AMD_IS_PX) && amdgpu_is_atpx_hybrid()))
 266                 return true;
 267         return false;
 268 }
 269
 270 /**
 271  * amdgpu_device_supports_baco - Does the device support BACO
 272  *
 273  * @dev: drm_device pointer
 274  *
 275  * Returns true if the device supporte BACO,
 276  * otherwise return false.
 277  */
 278 bool amdgpu_device_supports_baco(struct drm_device *dev)
 279 {
 280         struct amdgpu_device *adev = drm_to_adev(dev);
 281
 282         return amdgpu_asic_supports_baco(adev);
 283 }
 284
 285 /**
 286  * amdgpu_device_supports_smart_shift - Is the device dGPU with
 287  * smart shift support
 288  *
 289  * @dev: drm_device pointer
 290  *
 291  * Returns true if the device is a dGPU with Smart Shift support,
 292  * otherwise returns false.
 293  */
 294 bool amdgpu_device_supports_smart_shift(struct drm_device *dev)
 295 {
 296         return (amdgpu_device_supports_boco(dev) &&
 297                 amdgpu_acpi_is_power_shift_control_supported());
 298 }
 299
 300 /*
 301  * VRAM access helper functions
 302  */
 303
 304 /**
 305  * amdgpu_device_mm_access - access vram by MM_INDEX/MM_DATA
 306  *
 307  * @adev: amdgpu_device pointer
 308  * @pos: offset of the buffer in vram
 309  * @buf: virtual address of the buffer in system memory
 310  * @size: read/write size, sizeof(@buf) must > @size
 311  * @write: true - write to vram, otherwise - read from vram
 312  */
 313 void amdgpu_device_mm_access(struct amdgpu_device *adev, loff_t pos,
 314                              void *buf, size_t size, bool write)
 315 {
 316         unsigned long flags;
 317         uint32_t hi = ~0, tmp = 0;
 318         uint32_t *data = buf;
 319         uint64_t last;
 320         int idx;
 321
 322         if (!drm_dev_enter(adev_to_drm(adev), &idx))
 323                 return;
 324
 325         BUG_ON(!IS_ALIGNED(pos, 4) || !IS_ALIGNED(size, 4));
 326
 327         spin_lock_irqsave(&adev->mmio_idx_lock, flags);
 328         for (last = pos + size; pos < last; pos += 4) {
 329                 tmp = pos >> 31;
 330
 331                 WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)pos) | 0x80000000);
 332                 if (tmp != hi) {
 333                         WREG32_NO_KIQ(mmMM_INDEX_HI, tmp);
 334                         hi = tmp;
 335                 }
 336                 if (write)
 337                         WREG32_NO_KIQ(mmMM_DATA, *data++);
 338                 else
 339                         *data++ = RREG32_NO_KIQ(mmMM_DATA);
 340         }
 341
 342         spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
 343         drm_dev_exit(idx);
 344 }
 345
 346 /**
 347  * amdgpu_device_aper_access - access vram by vram aperature
 348  *
 349  * @adev: amdgpu_device pointer
 350  * @pos: offset of the buffer in vram
 351  * @buf: virtual address of the buffer in system memory
 352  * @size: read/write size, sizeof(@buf) must > @size
 353  * @write: true - write to vram, otherwise - read from vram
 354  *
 355  * The return value means how many bytes have been transferred.
 356  */
 357 size_t amdgpu_device_aper_access(struct amdgpu_device *adev, loff_t pos,
 358                                  void *buf, size_t size, bool write)
 359 {
 360 #ifdef CONFIG_64BIT
 361         void __iomem *addr;
 362         size_t count = 0;
 363         uint64_t last;
 364
 365         if (!adev->mman.aper_base_kaddr)
 366                 return 0;
 367
 368         last = min(pos + size, adev->gmc.visible_vram_size);
 369         if (last > pos) {
 370                 addr = adev->mman.aper_base_kaddr + pos;
 371                 count = last - pos;
 372
 373                 if (write) {
 374                         memcpy_toio(addr, buf, count);
 375                         /* Make sure HDP write cache flush happens without any reordering
 376                          * after the system memory contents are sent over PCIe device
 377                          */
 378                         mb();
 379                         amdgpu_device_flush_hdp(adev, NULL);
 380                 } else {
 381                         amdgpu_device_invalidate_hdp(adev, NULL);
 382                         /* Make sure HDP read cache is invalidated before issuing a read
 383                          * to the PCIe device
 384                          */
 385                         mb();
 386                         memcpy_fromio(buf, addr, count);
 387                 }
 388
 389         }
 390
 391         return count;
 392 #else
 393         return 0;
 394 #endif
 395 }
 396
 397 /**
 398  * amdgpu_device_vram_access - read/write a buffer in vram
 399  *
 400  * @adev: amdgpu_device pointer
 401  * @pos: offset of the buffer in vram
 402  * @buf: virtual address of the buffer in system memory
 403  * @size: read/write size, sizeof(@buf) must > @size
 404  * @write: true - write to vram, otherwise - read from vram
 405  */
 406 void amdgpu_device_vram_access(struct amdgpu_device *adev, loff_t pos,
 407                                void *buf, size_t size, bool write)
 408 {
 409         size_t count;
 410
 411         /* try to using vram apreature to access vram first */
 412         count = amdgpu_device_aper_access(adev, pos, buf, size, write);
 413         size -= count;
 414         if (size) {
 415                 /* using MM to access rest vram */
 416                 pos += count;
 417                 buf += count;
 418                 amdgpu_device_mm_access(adev, pos, buf, size, write);
 419         }
 420 }
 421
 422 /*
 423  * register access helper functions.
 424  */
 425
 426 /* Check if hw access should be skipped because of hotplug or device error */
 427 bool amdgpu_device_skip_hw_access(struct amdgpu_device *adev)
 428 {
 429         if (adev->no_hw_access)
 430                 return true;
 431
 432 #ifdef CONFIG_LOCKDEP
 433         /*
 434          * This is a bit complicated to understand, so worth a comment. What we assert
 435          * here is that the GPU reset is not running on another thread in parallel.
 436          *
 437          * For this we trylock the read side of the reset semaphore, if that succeeds
 438          * we know that the reset is not running in paralell.
 439          *
 440          * If the trylock fails we assert that we are either already holding the read
 441          * side of the lock or are the reset thread itself and hold the write side of
 442          * the lock.
 443          */
 444         if (in_task()) {
 445                 if (down_read_trylock(&adev->reset_domain->sem))
 446                         up_read(&adev->reset_domain->sem);
 447                 else
 448                         lockdep_assert_held(&adev->reset_domain->sem);
 449         }
 450 #endif
 451         return false;
 452 }
 453
 454 /**
 455  * amdgpu_device_rreg - read a memory mapped IO or indirect register
 456  *
 457  * @adev: amdgpu_device pointer
 458  * @reg: dword aligned register offset
 459  * @acc_flags: access flags which require special behavior
 460  *
 461  * Returns the 32 bit value from the offset specified.
 462  */
 463 uint32_t amdgpu_device_rreg(struct amdgpu_device *adev,
 464                             uint32_t reg, uint32_t acc_flags)
 465 {
 466         uint32_t ret;
 467
 468         if (amdgpu_device_skip_hw_access(adev))
 469                 return 0;
 470
 471         if ((reg * 4) < adev->rmmio_size) {
 472                 if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
 473                     amdgpu_sriov_runtime(adev) &&
 474                     down_read_trylock(&adev->reset_domain->sem)) {
 475                         ret = amdgpu_kiq_rreg(adev, reg);
 476                         up_read(&adev->reset_domain->sem);
 477                 } else {
 478                         ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
 479                 }
 480         } else {
 481                 ret = adev->pcie_rreg(adev, reg * 4);
 482         }
 483
 484         trace_amdgpu_device_rreg(adev->pdev->device, reg, ret);
 485
 486         return ret;
 487 }
 488
 489 /*
 490  * MMIO register read with bytes helper functions
 491  * @offset:bytes offset from MMIO start
 492  */
 493
 494 /**
 495  * amdgpu_mm_rreg8 - read a memory mapped IO register
 496  *
 497  * @adev: amdgpu_device pointer
 498  * @offset: byte aligned register offset
 499  *
 500  * Returns the 8 bit value from the offset specified.
 501  */
 502 uint8_t amdgpu_mm_rreg8(struct amdgpu_device *adev, uint32_t offset)
 503 {
 504         if (amdgpu_device_skip_hw_access(adev))
 505                 return 0;
 506
 507         if (offset < adev->rmmio_size)
 508                 return (readb(adev->rmmio + offset));
 509         BUG();
 510 }
 511
 512 /*
 513  * MMIO register write with bytes helper functions
 514  * @offset:bytes offset from MMIO start
 515  * @value: the value want to be written to the register
 516  */
 517
 518 /**
 519  * amdgpu_mm_wreg8 - read a memory mapped IO register
 520  *
 521  * @adev: amdgpu_device pointer
 522  * @offset: byte aligned register offset
 523  * @value: 8 bit value to write
 524  *
 525  * Writes the value specified to the offset specified.
 526  */
 527 void amdgpu_mm_wreg8(struct amdgpu_device *adev, uint32_t offset, uint8_t value)
 528 {
 529         if (amdgpu_device_skip_hw_access(adev))
 530                 return;
 531
 532         if (offset < adev->rmmio_size)
 533                 writeb(value, adev->rmmio + offset);
 534         else
 535                 BUG();
 536 }
 537
 538 /**
 539  * amdgpu_device_wreg - write to a memory mapped IO or indirect register
 540  *
 541  * @adev: amdgpu_device pointer
 542  * @reg: dword aligned register offset
 543  * @v: 32 bit value to write to the register
 544  * @acc_flags: access flags which require special behavior
 545  *
 546  * Writes the value specified to the offset specified.
 547  */
 548 void amdgpu_device_wreg(struct amdgpu_device *adev,
 549                         uint32_t reg, uint32_t v,
 550                         uint32_t acc_flags)
 551 {
 552         if (amdgpu_device_skip_hw_access(adev))
 553                 return;
 554
 555         if ((reg * 4) < adev->rmmio_size) {
 556                 if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
 557                     amdgpu_sriov_runtime(adev) &&
 558                     down_read_trylock(&adev->reset_domain->sem)) {
 559                         amdgpu_kiq_wreg(adev, reg, v);
 560                         up_read(&adev->reset_domain->sem);
 561                 } else {
 562                         writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
 563                 }
 564         } else {
 565                 adev->pcie_wreg(adev, reg * 4, v);
 566         }
 567
 568         trace_amdgpu_device_wreg(adev->pdev->device, reg, v);
 569 }
 570
 571 /**
 572  * amdgpu_mm_wreg_mmio_rlc -  write register either with direct/indirect mmio or with RLC path if in range
 573  *
 574  * @adev: amdgpu_device pointer
 575  * @reg: mmio/rlc register
 576  * @v: value to write
 577  * @xcc_id: xcc accelerated compute core id
 578  *
 579  * this function is invoked only for the debugfs register access
 580  */
 581 void amdgpu_mm_wreg_mmio_rlc(struct amdgpu_device *adev,
 582                              uint32_t reg, uint32_t v,
 583                              uint32_t xcc_id)
 584 {
 585         if (amdgpu_device_skip_hw_access(adev))
 586                 return;
 587
 588         if (amdgpu_sriov_fullaccess(adev) &&
 589             adev->gfx.rlc.funcs &&
 590             adev->gfx.rlc.funcs->is_rlcg_access_range) {
 591                 if (adev->gfx.rlc.funcs->is_rlcg_access_range(adev, reg))
 592                         return amdgpu_sriov_wreg(adev, reg, v, 0, 0, xcc_id);
 593         } else if ((reg * 4) >= adev->rmmio_size) {
 594                 adev->pcie_wreg(adev, reg * 4, v);
 595         } else {
 596                 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
 597         }
 598 }
 599
 600 /**
 601  * amdgpu_device_indirect_rreg - read an indirect register
 602  *
 603  * @adev: amdgpu_device pointer
 604  * @reg_addr: indirect register address to read from
 605  *
 606  * Returns the value of indirect register @reg_addr
 607  */
 608 u32 amdgpu_device_indirect_rreg(struct amdgpu_device *adev,
 609                                 u32 reg_addr)
 610 {
 611         unsigned long flags, pcie_index, pcie_data;
 612         void __iomem *pcie_index_offset;
 613         void __iomem *pcie_data_offset;
 614         u32 r;
 615
 616         pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
 617         pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
 618
 619         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 620         pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
 621         pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
 622
 623         writel(reg_addr, pcie_index_offset);
 624         readl(pcie_index_offset);
 625         r = readl(pcie_data_offset);
 626         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 627
 628         return r;
 629 }
 630
 631 u32 amdgpu_device_indirect_rreg_ext(struct amdgpu_device *adev,
 632                                     u64 reg_addr)
 633 {
 634         unsigned long flags, pcie_index, pcie_index_hi, pcie_data;
 635         u32 r;
 636         void __iomem *pcie_index_offset;
 637         void __iomem *pcie_index_hi_offset;
 638         void __iomem *pcie_data_offset;
 639
 640         pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
 641         pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
 642         if ((reg_addr >> 32) && (adev->nbio.funcs->get_pcie_index_hi_offset))
 643                 pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset(adev);
 644         else
 645                 pcie_index_hi = 0;
 646
 647         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 648         pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
 649         pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
 650         if (pcie_index_hi != 0)
 651                 pcie_index_hi_offset = (void __iomem *)adev->rmmio +
 652                                 pcie_index_hi * 4;
 653
 654         writel(reg_addr, pcie_index_offset);
 655         readl(pcie_index_offset);
 656         if (pcie_index_hi != 0) {
 657                 writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset);
 658                 readl(pcie_index_hi_offset);
 659         }
 660         r = readl(pcie_data_offset);
 661
 662         /* clear the high bits */
 663         if (pcie_index_hi != 0) {
 664                 writel(0, pcie_index_hi_offset);
 665                 readl(pcie_index_hi_offset);
 666         }
 667
 668         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 669
 670         return r;
 671 }
 672
 673 /**
 674  * amdgpu_device_indirect_rreg64 - read a 64bits indirect register
 675  *
 676  * @adev: amdgpu_device pointer
 677  * @reg_addr: indirect register address to read from
 678  *
 679  * Returns the value of indirect register @reg_addr
 680  */
 681 u64 amdgpu_device_indirect_rreg64(struct amdgpu_device *adev,
 682                                   u32 reg_addr)
 683 {
 684         unsigned long flags, pcie_index, pcie_data;
 685         void __iomem *pcie_index_offset;
 686         void __iomem *pcie_data_offset;
 687         u64 r;
 688
 689         pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
 690         pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
 691
 692         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 693         pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
 694         pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
 695
 696         /* read low 32 bits */
 697         writel(reg_addr, pcie_index_offset);
 698         readl(pcie_index_offset);
 699         r = readl(pcie_data_offset);
 700         /* read high 32 bits */
 701         writel(reg_addr + 4, pcie_index_offset);
 702         readl(pcie_index_offset);
 703         r |= ((u64)readl(pcie_data_offset) << 32);
 704         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 705
 706         return r;
 707 }
 708
 709 u64 amdgpu_device_indirect_rreg64_ext(struct amdgpu_device *adev,
 710                                   u64 reg_addr)
 711 {
 712         unsigned long flags, pcie_index, pcie_data;
 713         unsigned long pcie_index_hi = 0;
 714         void __iomem *pcie_index_offset;
 715         void __iomem *pcie_index_hi_offset;
 716         void __iomem *pcie_data_offset;
 717         u64 r;
 718
 719         pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
 720         pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
 721         if ((reg_addr >> 32) && (adev->nbio.funcs->get_pcie_index_hi_offset))
 722                 pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset(adev);
 723
 724         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 725         pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
 726         pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
 727         if (pcie_index_hi != 0)
 728                 pcie_index_hi_offset = (void __iomem *)adev->rmmio +
 729                         pcie_index_hi * 4;
 730
 731         /* read low 32 bits */
 732         writel(reg_addr, pcie_index_offset);
 733         readl(pcie_index_offset);
 734         if (pcie_index_hi != 0) {
 735                 writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset);
 736                 readl(pcie_index_hi_offset);
 737         }
 738         r = readl(pcie_data_offset);
 739         /* read high 32 bits */
 740         writel(reg_addr + 4, pcie_index_offset);
 741         readl(pcie_index_offset);
 742         if (pcie_index_hi != 0) {
 743                 writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset);
 744                 readl(pcie_index_hi_offset);
 745         }
 746         r |= ((u64)readl(pcie_data_offset) << 32);
 747
 748         /* clear the high bits */
 749         if (pcie_index_hi != 0) {
 750                 writel(0, pcie_index_hi_offset);
 751                 readl(pcie_index_hi_offset);
 752         }
 753
 754         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 755
 756         return r;
 757 }
 758
 759 /**
 760  * amdgpu_device_indirect_wreg - write an indirect register address
 761  *
 762  * @adev: amdgpu_device pointer
 763  * @reg_addr: indirect register offset
 764  * @reg_data: indirect register data
 765  *
 766  */
 767 void amdgpu_device_indirect_wreg(struct amdgpu_device *adev,
 768                                  u32 reg_addr, u32 reg_data)
 769 {
 770         unsigned long flags, pcie_index, pcie_data;
 771         void __iomem *pcie_index_offset;
 772         void __iomem *pcie_data_offset;
 773
 774         pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
 775         pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
 776
 777         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 778         pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
 779         pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
 780
 781         writel(reg_addr, pcie_index_offset);
 782         readl(pcie_index_offset);
 783         writel(reg_data, pcie_data_offset);
 784         readl(pcie_data_offset);
 785         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 786 }
 787
 788 void amdgpu_device_indirect_wreg_ext(struct amdgpu_device *adev,
 789                                      u64 reg_addr, u32 reg_data)
 790 {
 791         unsigned long flags, pcie_index, pcie_index_hi, pcie_data;
 792         void __iomem *pcie_index_offset;
 793         void __iomem *pcie_index_hi_offset;
 794         void __iomem *pcie_data_offset;
 795
 796         pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
 797         pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
 798         if ((reg_addr >> 32) && (adev->nbio.funcs->get_pcie_index_hi_offset))
 799                 pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset(adev);
 800         else
 801                 pcie_index_hi = 0;
 802
 803         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 804         pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
 805         pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
 806         if (pcie_index_hi != 0)
 807                 pcie_index_hi_offset = (void __iomem *)adev->rmmio +
 808                                 pcie_index_hi * 4;
 809
 810         writel(reg_addr, pcie_index_offset);
 811         readl(pcie_index_offset);
 812         if (pcie_index_hi != 0) {
 813                 writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset);
 814                 readl(pcie_index_hi_offset);
 815         }
 816         writel(reg_data, pcie_data_offset);
 817         readl(pcie_data_offset);
 818
 819         /* clear the high bits */
 820         if (pcie_index_hi != 0) {
 821                 writel(0, pcie_index_hi_offset);
 822                 readl(pcie_index_hi_offset);
 823         }
 824
 825         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 826 }
 827
 828 /**
 829  * amdgpu_device_indirect_wreg64 - write a 64bits indirect register address
 830  *
 831  * @adev: amdgpu_device pointer
 832  * @reg_addr: indirect register offset
 833  * @reg_data: indirect register data
 834  *
 835  */
 836 void amdgpu_device_indirect_wreg64(struct amdgpu_device *adev,
 837                                    u32 reg_addr, u64 reg_data)
 838 {
 839         unsigned long flags, pcie_index, pcie_data;
 840         void __iomem *pcie_index_offset;
 841         void __iomem *pcie_data_offset;
 842
 843         pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
 844         pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
 845
 846         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 847         pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
 848         pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
 849
 850         /* write low 32 bits */
 851         writel(reg_addr, pcie_index_offset);
 852         readl(pcie_index_offset);
 853         writel((u32)(reg_data & 0xffffffffULL), pcie_data_offset);
 854         readl(pcie_data_offset);
 855         /* write high 32 bits */
 856         writel(reg_addr + 4, pcie_index_offset);
 857         readl(pcie_index_offset);
 858         writel((u32)(reg_data >> 32), pcie_data_offset);
 859         readl(pcie_data_offset);
 860         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 861 }
 862
 863 void amdgpu_device_indirect_wreg64_ext(struct amdgpu_device *adev,
 864                                    u64 reg_addr, u64 reg_data)
 865 {
 866         unsigned long flags, pcie_index, pcie_data;
 867         unsigned long pcie_index_hi = 0;
 868         void __iomem *pcie_index_offset;
 869         void __iomem *pcie_index_hi_offset;
 870         void __iomem *pcie_data_offset;
 871
 872         pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
 873         pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
 874         if ((reg_addr >> 32) && (adev->nbio.funcs->get_pcie_index_hi_offset))
 875                 pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset(adev);
 876
 877         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 878         pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
 879         pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
 880         if (pcie_index_hi != 0)
 881                 pcie_index_hi_offset = (void __iomem *)adev->rmmio +
 882                                 pcie_index_hi * 4;
 883
 884         /* write low 32 bits */
 885         writel(reg_addr, pcie_index_offset);
 886         readl(pcie_index_offset);
 887         if (pcie_index_hi != 0) {
 888                 writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset);
 889                 readl(pcie_index_hi_offset);
 890         }
 891         writel((u32)(reg_data & 0xffffffffULL), pcie_data_offset);
 892         readl(pcie_data_offset);
 893         /* write high 32 bits */
 894         writel(reg_addr + 4, pcie_index_offset);
 895         readl(pcie_index_offset);
 896         if (pcie_index_hi != 0) {
 897                 writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset);
 898                 readl(pcie_index_hi_offset);
 899         }
 900         writel((u32)(reg_data >> 32), pcie_data_offset);
 901         readl(pcie_data_offset);
 902
 903         /* clear the high bits */
 904         if (pcie_index_hi != 0) {
 905                 writel(0, pcie_index_hi_offset);
 906                 readl(pcie_index_hi_offset);
 907         }
 908
 909         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 910 }
 911
 912 /**
 913  * amdgpu_device_get_rev_id - query device rev_id
 914  *
 915  * @adev: amdgpu_device pointer
 916  *
 917  * Return device rev_id
 918  */
 919 u32 amdgpu_device_get_rev_id(struct amdgpu_device *adev)
 920 {
 921         return adev->nbio.funcs->get_rev_id(adev);
 922 }
 923
 924 /**
 925  * amdgpu_invalid_rreg - dummy reg read function
 926  *
 927  * @adev: amdgpu_device pointer
 928  * @reg: offset of register
 929  *
 930  * Dummy register read function.  Used for register blocks
 931  * that certain asics don't have (all asics).
 932  * Returns the value in the register.
 933  */
 934 static uint32_t amdgpu_invalid_rreg(struct amdgpu_device *adev, uint32_t reg)
 935 {
 936         DRM_ERROR("Invalid callback to read register 0x%04X\n", reg);
 937         BUG();
 938         return 0;
 939 }
 940
 941 static uint32_t amdgpu_invalid_rreg_ext(struct amdgpu_device *adev, uint64_t reg)
 942 {
 943         DRM_ERROR("Invalid callback to read register 0x%llX\n", reg);
 944         BUG();
 945         return 0;
 946 }
 947
 948 /**
 949  * amdgpu_invalid_wreg - dummy reg write function
 950  *
 951  * @adev: amdgpu_device pointer
 952  * @reg: offset of register
 953  * @v: value to write to the register
 954  *
 955  * Dummy register read function.  Used for register blocks
 956  * that certain asics don't have (all asics).
 957  */
 958 static void amdgpu_invalid_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
 959 {
 960         DRM_ERROR("Invalid callback to write register 0x%04X with 0x%08X\n",
 961                   reg, v);
 962         BUG();
 963 }
 964
 965 static void amdgpu_invalid_wreg_ext(struct amdgpu_device *adev, uint64_t reg, uint32_t v)
 966 {
 967         DRM_ERROR("Invalid callback to write register 0x%llX with 0x%08X\n",
 968                   reg, v);
 969         BUG();
 970 }
 971
 972 /**
 973  * amdgpu_invalid_rreg64 - dummy 64 bit reg read function
 974  *
 975  * @adev: amdgpu_device pointer
 976  * @reg: offset of register
 977  *
 978  * Dummy register read function.  Used for register blocks
 979  * that certain asics don't have (all asics).
 980  * Returns the value in the register.
 981  */
 982 static uint64_t amdgpu_invalid_rreg64(struct amdgpu_device *adev, uint32_t reg)
 983 {
 984         DRM_ERROR("Invalid callback to read 64 bit register 0x%04X\n", reg);
 985         BUG();
 986         return 0;
 987 }
 988
 989 static uint64_t amdgpu_invalid_rreg64_ext(struct amdgpu_device *adev, uint64_t reg)
 990 {
 991         DRM_ERROR("Invalid callback to read register 0x%llX\n", reg);
 992         BUG();
 993         return 0;
 994 }
 995
 996 /**
 997  * amdgpu_invalid_wreg64 - dummy reg write function
 998  *
 999  * @adev: amdgpu_device pointer
1000  * @reg: offset of register
1001  * @v: value to write to the register
1002  *
1003  * Dummy register read function.  Used for register blocks
1004  * that certain asics don't have (all asics).
1005  */
1006 static void amdgpu_invalid_wreg64(struct amdgpu_device *adev, uint32_t reg, uint64_t v)
1007 {
1008         DRM_ERROR("Invalid callback to write 64 bit register 0x%04X with 0x%08llX\n",
1009                   reg, v);
1010         BUG();
1011 }
1012
1013 static void amdgpu_invalid_wreg64_ext(struct amdgpu_device *adev, uint64_t reg, uint64_t v)
1014 {
1015         DRM_ERROR("Invalid callback to write 64 bit register 0x%llX with 0x%08llX\n",
1016                   reg, v);
1017         BUG();
1018 }
1019
1020 /**
1021  * amdgpu_block_invalid_rreg - dummy reg read function
1022  *
1023  * @adev: amdgpu_device pointer
1024  * @block: offset of instance
1025  * @reg: offset of register
1026  *
1027  * Dummy register read function.  Used for register blocks
1028  * that certain asics don't have (all asics).
1029  * Returns the value in the register.
1030  */
1031 static uint32_t amdgpu_block_invalid_rreg(struct amdgpu_device *adev,
1032                                           uint32_t block, uint32_t reg)
1033 {
1034         DRM_ERROR("Invalid callback to read register 0x%04X in block 0x%04X\n",
1035                   reg, block);
1036         BUG();
1037         return 0;
1038 }
1039
1040 /**
1041  * amdgpu_block_invalid_wreg - dummy reg write function
1042  *
1043  * @adev: amdgpu_device pointer
1044  * @block: offset of instance
1045  * @reg: offset of register
1046  * @v: value to write to the register
1047  *
1048  * Dummy register read function.  Used for register blocks
1049  * that certain asics don't have (all asics).
1050  */
1051 static void amdgpu_block_invalid_wreg(struct amdgpu_device *adev,
1052                                       uint32_t block,
1053                                       uint32_t reg, uint32_t v)
1054 {
1055         DRM_ERROR("Invalid block callback to write register 0x%04X in block 0x%04X with 0x%08X\n",
1056                   reg, block, v);
1057         BUG();
1058 }
1059
1060 /**
1061  * amdgpu_device_asic_init - Wrapper for atom asic_init
1062  *
1063  * @adev: amdgpu_device pointer
1064  *
1065  * Does any asic specific work and then calls atom asic init.
1066  */
1067 static int amdgpu_device_asic_init(struct amdgpu_device *adev)
1068 {
1069         int ret;
1070
1071         amdgpu_asic_pre_asic_init(adev);
1072
1073         if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 3) ||
1074             amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(11, 0, 0)) {
1075                 amdgpu_psp_wait_for_bootloader(adev);
1076                 ret = amdgpu_atomfirmware_asic_init(adev, true);
1077                 /* TODO: check the return val and stop device initialization if boot fails */
1078                 amdgpu_psp_query_boot_status(adev);
1079                 return ret;
1080         } else {
1081                 return amdgpu_atom_asic_init(adev->mode_info.atom_context);
1082         }
1083
1084         return 0;
1085 }
1086
1087 /**
1088  * amdgpu_device_mem_scratch_init - allocate the VRAM scratch page
1089  *
1090  * @adev: amdgpu_device pointer
1091  *
1092  * Allocates a scratch page of VRAM for use by various things in the
1093  * driver.
1094  */
1095 static int amdgpu_device_mem_scratch_init(struct amdgpu_device *adev)
1096 {
1097         return amdgpu_bo_create_kernel(adev, AMDGPU_GPU_PAGE_SIZE, PAGE_SIZE,
1098                                        AMDGPU_GEM_DOMAIN_VRAM |
1099                                        AMDGPU_GEM_DOMAIN_GTT,
1100                                        &adev->mem_scratch.robj,
1101                                        &adev->mem_scratch.gpu_addr,
1102                                        (void **)&adev->mem_scratch.ptr);
1103 }
1104
1105 /**
1106  * amdgpu_device_mem_scratch_fini - Free the VRAM scratch page
1107  *
1108  * @adev: amdgpu_device pointer
1109  *
1110  * Frees the VRAM scratch page.
1111  */
1112 static void amdgpu_device_mem_scratch_fini(struct amdgpu_device *adev)
1113 {
1114         amdgpu_bo_free_kernel(&adev->mem_scratch.robj, NULL, NULL);
1115 }
1116
1117 /**
1118  * amdgpu_device_program_register_sequence - program an array of registers.
1119  *
1120  * @adev: amdgpu_device pointer
1121  * @registers: pointer to the register array
1122  * @array_size: size of the register array
1123  *
1124  * Programs an array or registers with and or masks.
1125  * This is a helper for setting golden registers.
1126  */
1127 void amdgpu_device_program_register_sequence(struct amdgpu_device *adev,
1128                                              const u32 *registers,
1129                                              const u32 array_size)
1130 {
1131         u32 tmp, reg, and_mask, or_mask;
1132         int i;
1133
1134         if (array_size % 3)
1135                 return;
1136
1137         for (i = 0; i < array_size; i += 3) {
1138                 reg = registers[i + 0];
1139                 and_mask = registers[i + 1];
1140                 or_mask = registers[i + 2];
1141
1142                 if (and_mask == 0xffffffff) {
1143                         tmp = or_mask;
1144                 } else {
1145                         tmp = RREG32(reg);
1146                         tmp &= ~and_mask;
1147                         if (adev->family >= AMDGPU_FAMILY_AI)
1148                                 tmp |= (or_mask & and_mask);
1149                         else
1150                                 tmp |= or_mask;
1151                 }
1152                 WREG32(reg, tmp);
1153         }
1154 }
1155
1156 /**
1157  * amdgpu_device_pci_config_reset - reset the GPU
1158  *
1159  * @adev: amdgpu_device pointer
1160  *
1161  * Resets the GPU using the pci config reset sequence.
1162  * Only applicable to asics prior to vega10.
1163  */
1164 void amdgpu_device_pci_config_reset(struct amdgpu_device *adev)
1165 {
1166         pci_write_config_dword(adev->pdev, 0x7c, AMDGPU_ASIC_RESET_DATA);
1167 }
1168
1169 /**
1170  * amdgpu_device_pci_reset - reset the GPU using generic PCI means
1171  *
1172  * @adev: amdgpu_device pointer
1173  *
1174  * Resets the GPU using generic pci reset interfaces (FLR, SBR, etc.).
1175  */
1176 int amdgpu_device_pci_reset(struct amdgpu_device *adev)
1177 {
1178         return pci_reset_function(adev->pdev);
1179 }
1180
1181 /*
1182  * amdgpu_device_wb_*()
1183  * Writeback is the method by which the GPU updates special pages in memory
1184  * with the status of certain GPU events (fences, ring pointers,etc.).
1185  */
1186
1187 /**
1188  * amdgpu_device_wb_fini - Disable Writeback and free memory
1189  *
1190  * @adev: amdgpu_device pointer
1191  *
1192  * Disables Writeback and frees the Writeback memory (all asics).
1193  * Used at driver shutdown.
1194  */
1195 static void amdgpu_device_wb_fini(struct amdgpu_device *adev)
1196 {
1197         if (adev->wb.wb_obj) {
1198                 amdgpu_bo_free_kernel(&adev->wb.wb_obj,
1199                                       &adev->wb.gpu_addr,
1200                                       (void **)&adev->wb.wb);
1201                 adev->wb.wb_obj = NULL;
1202         }
1203 }
1204
1205 /**
1206  * amdgpu_device_wb_init - Init Writeback driver info and allocate memory
1207  *
1208  * @adev: amdgpu_device pointer
1209  *
1210  * Initializes writeback and allocates writeback memory (all asics).
1211  * Used at driver startup.
1212  * Returns 0 on success or an -error on failure.
1213  */
1214 static int amdgpu_device_wb_init(struct amdgpu_device *adev)
1215 {
1216         int r;
1217
1218         if (adev->wb.wb_obj == NULL) {
1219                 /* AMDGPU_MAX_WB * sizeof(uint32_t) * 8 = AMDGPU_MAX_WB 256bit slots */
1220                 r = amdgpu_bo_create_kernel(adev, AMDGPU_MAX_WB * sizeof(uint32_t) * 8,
1221                                             PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
1222                                             &adev->wb.wb_obj, &adev->wb.gpu_addr,
1223                                             (void **)&adev->wb.wb);
1224                 if (r) {
1225                         dev_warn(adev->dev, "(%d) create WB bo failed\n", r);
1226                         return r;
1227                 }
1228
1229                 adev->wb.num_wb = AMDGPU_MAX_WB;
1230                 memset(&adev->wb.used, 0, sizeof(adev->wb.used));
1231
1232                 /* clear wb memory */
1233                 memset((char *)adev->wb.wb, 0, AMDGPU_MAX_WB * sizeof(uint32_t) * 8);
1234         }
1235
1236         return 0;
1237 }
1238
1239 /**
1240  * amdgpu_device_wb_get - Allocate a wb entry
1241  *
1242  * @adev: amdgpu_device pointer
1243  * @wb: wb index
1244  *
1245  * Allocate a wb slot for use by the driver (all asics).
1246  * Returns 0 on success or -EINVAL on failure.
1247  */
1248 int amdgpu_device_wb_get(struct amdgpu_device *adev, u32 *wb)
1249 {
1250         unsigned long offset = find_first_zero_bit(adev->wb.used, adev->wb.num_wb);
1251
1252         if (offset < adev->wb.num_wb) {
1253                 __set_bit(offset, adev->wb.used);
1254                 *wb = offset << 3; /* convert to dw offset */
1255                 return 0;
1256         } else {
1257                 return -EINVAL;
1258         }
1259 }
1260
1261 /**
1262  * amdgpu_device_wb_free - Free a wb entry
1263  *
1264  * @adev: amdgpu_device pointer
1265  * @wb: wb index
1266  *
1267  * Free a wb slot allocated for use by the driver (all asics)
1268  */
1269 void amdgpu_device_wb_free(struct amdgpu_device *adev, u32 wb)
1270 {
1271         wb >>= 3;
1272         if (wb < adev->wb.num_wb)
1273                 __clear_bit(wb, adev->wb.used);
1274 }
1275
1276 /**
1277  * amdgpu_device_resize_fb_bar - try to resize FB BAR
1278  *
1279  * @adev: amdgpu_device pointer
1280  *
1281  * Try to resize FB BAR to make all VRAM CPU accessible. We try very hard not
1282  * to fail, but if any of the BARs is not accessible after the size we abort
1283  * driver loading by returning -ENODEV.
1284  */
1285 int amdgpu_device_resize_fb_bar(struct amdgpu_device *adev)
1286 {
1287         int rbar_size = pci_rebar_bytes_to_size(adev->gmc.real_vram_size);
1288         struct pci_bus *root;
1289         struct resource *res;
1290         unsigned int i;
1291         u16 cmd;
1292         int r;
1293
1294         if (!IS_ENABLED(CONFIG_PHYS_ADDR_T_64BIT))
1295                 return 0;
1296
1297         /* Bypass for VF */
1298         if (amdgpu_sriov_vf(adev))
1299                 return 0;
1300
1301         /* skip if the bios has already enabled large BAR */
1302         if (adev->gmc.real_vram_size &&
1303             (pci_resource_len(adev->pdev, 0) >= adev->gmc.real_vram_size))
1304                 return 0;
1305
1306         /* Check if the root BUS has 64bit memory resources */
1307         root = adev->pdev->bus;
1308         while (root->parent)
1309                 root = root->parent;
1310
1311         pci_bus_for_each_resource(root, res, i) {
1312                 if (res && res->flags & (IORESOURCE_MEM | IORESOURCE_MEM_64) &&
1313                     res->start > 0x100000000ull)
1314                         break;
1315         }
1316
1317         /* Trying to resize is pointless without a root hub window above 4GB */
1318         if (!res)
1319                 return 0;
1320
1321         /* Limit the BAR size to what is available */
1322         rbar_size = min(fls(pci_rebar_get_possible_sizes(adev->pdev, 0)) - 1,
1323                         rbar_size);
1324
1325         /* Disable memory decoding while we change the BAR addresses and size */
1326         pci_read_config_word(adev->pdev, PCI_COMMAND, &cmd);
1327         pci_write_config_word(adev->pdev, PCI_COMMAND,
1328                               cmd & ~PCI_COMMAND_MEMORY);
1329
1330         /* Free the VRAM and doorbell BAR, we most likely need to move both. */
1331         amdgpu_doorbell_fini(adev);
1332         if (adev->asic_type >= CHIP_BONAIRE)
1333                 pci_release_resource(adev->pdev, 2);
1334
1335         pci_release_resource(adev->pdev, 0);
1336
1337         r = pci_resize_resource(adev->pdev, 0, rbar_size);
1338         if (r == -ENOSPC)
1339                 DRM_INFO("Not enough PCI address space for a large BAR.");
1340         else if (r && r != -ENOTSUPP)
1341                 DRM_ERROR("Problem resizing BAR0 (%d).", r);
1342
1343         pci_assign_unassigned_bus_resources(adev->pdev->bus);
1344
1345         /* When the doorbell or fb BAR isn't available we have no chance of
1346          * using the device.
1347          */
1348         r = amdgpu_doorbell_init(adev);
1349         if (r || (pci_resource_flags(adev->pdev, 0) & IORESOURCE_UNSET))
1350                 return -ENODEV;
1351
1352         pci_write_config_word(adev->pdev, PCI_COMMAND, cmd);
1353
1354         return 0;
1355 }
1356
1357 static bool amdgpu_device_read_bios(struct amdgpu_device *adev)
1358 {
1359         if (hweight32(adev->aid_mask) && (adev->flags & AMD_IS_APU))
1360                 return false;
1361
1362         return true;
1363 }
1364
1365 /*
1366  * GPU helpers function.
1367  */
1368 /**
1369  * amdgpu_device_need_post - check if the hw need post or not
1370  *
1371  * @adev: amdgpu_device pointer
1372  *
1373  * Check if the asic has been initialized (all asics) at driver startup
1374  * or post is needed if  hw reset is performed.
1375  * Returns true if need or false if not.
1376  */
1377 bool amdgpu_device_need_post(struct amdgpu_device *adev)
1378 {
1379         uint32_t reg;
1380
1381         if (amdgpu_sriov_vf(adev))
1382                 return false;
1383
1384         if (!amdgpu_device_read_bios(adev))
1385                 return false;
1386
1387         if (amdgpu_passthrough(adev)) {
1388                 /* for FIJI: In whole GPU pass-through virtualization case, after VM reboot
1389                  * some old smc fw still need driver do vPost otherwise gpu hang, while
1390                  * those smc fw version above 22.15 doesn't have this flaw, so we force
1391                  * vpost executed for smc version below 22.15
1392                  */
1393                 if (adev->asic_type == CHIP_FIJI) {
1394                         int err;
1395                         uint32_t fw_ver;
1396
1397                         err = request_firmware(&adev->pm.fw, "amdgpu/fiji_smc.bin", adev->dev);
1398                         /* force vPost if error occured */
1399                         if (err)
1400                                 return true;
1401
1402                         fw_ver = *((uint32_t *)adev->pm.fw->data + 69);
1403                         if (fw_ver < 0x00160e00)
1404                                 return true;
1405                 }
1406         }
1407
1408         /* Don't post if we need to reset whole hive on init */
1409         if (adev->gmc.xgmi.pending_reset)
1410                 return false;
1411
1412         if (adev->has_hw_reset) {
1413                 adev->has_hw_reset = false;
1414                 return true;
1415         }
1416
1417         /* bios scratch used on CIK+ */
1418         if (adev->asic_type >= CHIP_BONAIRE)
1419                 return amdgpu_atombios_scratch_need_asic_init(adev);
1420
1421         /* check MEM_SIZE for older asics */
1422         reg = amdgpu_asic_get_config_memsize(adev);
1423
1424         if ((reg != 0) && (reg != 0xffffffff))
1425                 return false;
1426
1427         return true;
1428 }
1429
1430 /*
1431  * Check whether seamless boot is supported.
1432  *
1433  * So far we only support seamless boot on DCE 3.0 or later.
1434  * If users report that it works on older ASICS as well, we may
1435  * loosen this.
1436  */
1437 bool amdgpu_device_seamless_boot_supported(struct amdgpu_device *adev)
1438 {
1439         switch (amdgpu_seamless) {
1440         case -1:
1441                 break;
1442         case 1:
1443                 return true;
1444         case 0:
1445                 return false;
1446         default:
1447                 DRM_ERROR("Invalid value for amdgpu.seamless: %d\n",
1448                           amdgpu_seamless);
1449                 return false;
1450         }
1451
1452         if (!(adev->flags & AMD_IS_APU))
1453                 return false;
1454
1455         if (adev->mman.keep_stolen_vga_memory)
1456                 return false;
1457
1458         return adev->ip_versions[DCE_HWIP][0] >= IP_VERSION(3, 0, 0);
1459 }
1460
1461 /*
1462  * Intel hosts such as Rocket Lake, Alder Lake, Raptor Lake and Sapphire Rapids
1463  * don't support dynamic speed switching. Until we have confirmation from Intel
1464  * that a specific host supports it, it's safer that we keep it disabled for all.
1465  *
1466  * https://edc.intel.com/content/www/us/en/design/products/platforms/details/raptor-lake-s/13th-generation-core-processors-datasheet-volume-1-of-2/005/pci-express-support/
1467  * https://gitlab.freedesktop.org/drm/amd/-/issues/2663
1468  */
1469 static bool amdgpu_device_pcie_dynamic_switching_supported(void)
1470 {
1471 #if IS_ENABLED(CONFIG_X86)
1472         struct cpuinfo_x86 *c = &cpu_data(0);
1473
1474         if (c->x86_vendor == X86_VENDOR_INTEL)
1475                 return false;
1476 #endif
1477         return true;
1478 }
1479
1480 /**
1481  * amdgpu_device_should_use_aspm - check if the device should program ASPM
1482  *
1483  * @adev: amdgpu_device pointer
1484  *
1485  * Confirm whether the module parameter and pcie bridge agree that ASPM should
1486  * be set for this device.
1487  *
1488  * Returns true if it should be used or false if not.
1489  */
1490 bool amdgpu_device_should_use_aspm(struct amdgpu_device *adev)
1491 {
1492         switch (amdgpu_aspm) {
1493         case -1:
1494                 break;
1495         case 0:
1496                 return false;
1497         case 1:
1498                 return true;
1499         default:
1500                 return false;
1501         }
1502         if (adev->flags & AMD_IS_APU)
1503                 return false;
1504         if (!(adev->pm.pp_feature & PP_PCIE_DPM_MASK))
1505                 return false;
1506         return pcie_aspm_enabled(adev->pdev);
1507 }
1508
1509 /* if we get transitioned to only one device, take VGA back */
1510 /**
1511  * amdgpu_device_vga_set_decode - enable/disable vga decode
1512  *
1513  * @pdev: PCI device pointer
1514  * @state: enable/disable vga decode
1515  *
1516  * Enable/disable vga decode (all asics).
1517  * Returns VGA resource flags.
1518  */
1519 static unsigned int amdgpu_device_vga_set_decode(struct pci_dev *pdev,
1520                 bool state)
1521 {
1522         struct amdgpu_device *adev = drm_to_adev(pci_get_drvdata(pdev));
1523
1524         amdgpu_asic_set_vga_state(adev, state);
1525         if (state)
1526                 return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM |
1527                        VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
1528         else
1529                 return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
1530 }
1531
1532 /**
1533  * amdgpu_device_check_block_size - validate the vm block size
1534  *
1535  * @adev: amdgpu_device pointer
1536  *
1537  * Validates the vm block size specified via module parameter.
1538  * The vm block size defines number of bits in page table versus page directory,
1539  * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
1540  * page table and the remaining bits are in the page directory.
1541  */
1542 static void amdgpu_device_check_block_size(struct amdgpu_device *adev)
1543 {
1544         /* defines number of bits in page table versus page directory,
1545          * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
1546          * page table and the remaining bits are in the page directory
1547          */
1548         if (amdgpu_vm_block_size == -1)
1549                 return;
1550
1551         if (amdgpu_vm_block_size < 9) {
1552                 dev_warn(adev->dev, "VM page table size (%d) too small\n",
1553                          amdgpu_vm_block_size);
1554                 amdgpu_vm_block_size = -1;
1555         }
1556 }
1557
1558 /**
1559  * amdgpu_device_check_vm_size - validate the vm size
1560  *
1561  * @adev: amdgpu_device pointer
1562  *
1563  * Validates the vm size in GB specified via module parameter.
1564  * The VM size is the size of the GPU virtual memory space in GB.
1565  */
1566 static void amdgpu_device_check_vm_size(struct amdgpu_device *adev)
1567 {
1568         /* no need to check the default value */
1569         if (amdgpu_vm_size == -1)
1570                 return;
1571
1572         if (amdgpu_vm_size < 1) {
1573                 dev_warn(adev->dev, "VM size (%d) too small, min is 1GB\n",
1574                          amdgpu_vm_size);
1575                 amdgpu_vm_size = -1;
1576         }
1577 }
1578
1579 static void amdgpu_device_check_smu_prv_buffer_size(struct amdgpu_device *adev)
1580 {
1581         struct sysinfo si;
1582         bool is_os_64 = (sizeof(void *) == 8);
1583         uint64_t total_memory;
1584         uint64_t dram_size_seven_GB = 0x1B8000000;
1585         uint64_t dram_size_three_GB = 0xB8000000;
1586
1587         if (amdgpu_smu_memory_pool_size == 0)
1588                 return;
1589
1590         if (!is_os_64) {
1591                 DRM_WARN("Not 64-bit OS, feature not supported\n");
1592                 goto def_value;
1593         }
1594         si_meminfo(&si);
1595         total_memory = (uint64_t)si.totalram * si.mem_unit;
1596
1597         if ((amdgpu_smu_memory_pool_size == 1) ||
1598                 (amdgpu_smu_memory_pool_size == 2)) {
1599                 if (total_memory < dram_size_three_GB)
1600                         goto def_value1;
1601         } else if ((amdgpu_smu_memory_pool_size == 4) ||
1602                 (amdgpu_smu_memory_pool_size == 8)) {
1603                 if (total_memory < dram_size_seven_GB)
1604                         goto def_value1;
1605         } else {
1606                 DRM_WARN("Smu memory pool size not supported\n");
1607                 goto def_value;
1608         }
1609         adev->pm.smu_prv_buffer_size = amdgpu_smu_memory_pool_size << 28;
1610
1611         return;
1612
1613 def_value1:
1614         DRM_WARN("No enough system memory\n");
1615 def_value:
1616         adev->pm.smu_prv_buffer_size = 0;
1617 }
1618
1619 static int amdgpu_device_init_apu_flags(struct amdgpu_device *adev)
1620 {
1621         if (!(adev->flags & AMD_IS_APU) ||
1622             adev->asic_type < CHIP_RAVEN)
1623                 return 0;
1624
1625         switch (adev->asic_type) {
1626         case CHIP_RAVEN:
1627                 if (adev->pdev->device == 0x15dd)
1628                         adev->apu_flags |= AMD_APU_IS_RAVEN;
1629                 if (adev->pdev->device == 0x15d8)
1630                         adev->apu_flags |= AMD_APU_IS_PICASSO;
1631                 break;
1632         case CHIP_RENOIR:
1633                 if ((adev->pdev->device == 0x1636) ||
1634                     (adev->pdev->device == 0x164c))
1635                         adev->apu_flags |= AMD_APU_IS_RENOIR;
1636                 else
1637                         adev->apu_flags |= AMD_APU_IS_GREEN_SARDINE;
1638                 break;
1639         case CHIP_VANGOGH:
1640                 adev->apu_flags |= AMD_APU_IS_VANGOGH;
1641                 break;
1642         case CHIP_YELLOW_CARP:
1643                 break;
1644         case CHIP_CYAN_SKILLFISH:
1645                 if ((adev->pdev->device == 0x13FE) ||
1646                     (adev->pdev->device == 0x143F))
1647                         adev->apu_flags |= AMD_APU_IS_CYAN_SKILLFISH2;
1648                 break;
1649         default:
1650                 break;
1651         }
1652
1653         return 0;
1654 }
1655
1656 /**
1657  * amdgpu_device_check_arguments - validate module params
1658  *
1659  * @adev: amdgpu_device pointer
1660  *
1661  * Validates certain module parameters and updates
1662  * the associated values used by the driver (all asics).
1663  */
1664 static int amdgpu_device_check_arguments(struct amdgpu_device *adev)
1665 {
1666         if (amdgpu_sched_jobs < 4) {
1667                 dev_warn(adev->dev, "sched jobs (%d) must be at least 4\n",
1668                          amdgpu_sched_jobs);
1669                 amdgpu_sched_jobs = 4;
1670         } else if (!is_power_of_2(amdgpu_sched_jobs)) {
1671                 dev_warn(adev->dev, "sched jobs (%d) must be a power of 2\n",
1672                          amdgpu_sched_jobs);
1673                 amdgpu_sched_jobs = roundup_pow_of_two(amdgpu_sched_jobs);
1674         }
1675
1676         if (amdgpu_gart_size != -1 && amdgpu_gart_size < 32) {
1677                 /* gart size must be greater or equal to 32M */
1678                 dev_warn(adev->dev, "gart size (%d) too small\n",
1679                          amdgpu_gart_size);
1680                 amdgpu_gart_size = -1;
1681         }
1682
1683         if (amdgpu_gtt_size != -1 && amdgpu_gtt_size < 32) {
1684                 /* gtt size must be greater or equal to 32M */
1685                 dev_warn(adev->dev, "gtt size (%d) too small\n",
1686                                  amdgpu_gtt_size);
1687                 amdgpu_gtt_size = -1;
1688         }
1689
1690         /* valid range is between 4 and 9 inclusive */
1691         if (amdgpu_vm_fragment_size != -1 &&
1692             (amdgpu_vm_fragment_size > 9 || amdgpu_vm_fragment_size < 4)) {
1693                 dev_warn(adev->dev, "valid range is between 4 and 9\n");
1694                 amdgpu_vm_fragment_size = -1;
1695         }
1696
1697         if (amdgpu_sched_hw_submission < 2) {
1698                 dev_warn(adev->dev, "sched hw submission jobs (%d) must be at least 2\n",
1699                          amdgpu_sched_hw_submission);
1700                 amdgpu_sched_hw_submission = 2;
1701         } else if (!is_power_of_2(amdgpu_sched_hw_submission)) {
1702                 dev_warn(adev->dev, "sched hw submission jobs (%d) must be a power of 2\n",
1703                          amdgpu_sched_hw_submission);
1704                 amdgpu_sched_hw_submission = roundup_pow_of_two(amdgpu_sched_hw_submission);
1705         }
1706
1707         if (amdgpu_reset_method < -1 || amdgpu_reset_method > 4) {
1708                 dev_warn(adev->dev, "invalid option for reset method, reverting to default\n");
1709                 amdgpu_reset_method = -1;
1710         }
1711
1712         amdgpu_device_check_smu_prv_buffer_size(adev);
1713
1714         amdgpu_device_check_vm_size(adev);
1715
1716         amdgpu_device_check_block_size(adev);
1717
1718         adev->firmware.load_type = amdgpu_ucode_get_load_type(adev, amdgpu_fw_load_type);
1719
1720         return 0;
1721 }
1722
1723 /**
1724  * amdgpu_switcheroo_set_state - set switcheroo state
1725  *
1726  * @pdev: pci dev pointer
1727  * @state: vga_switcheroo state
1728  *
1729  * Callback for the switcheroo driver.  Suspends or resumes
1730  * the asics before or after it is powered up using ACPI methods.
1731  */
1732 static void amdgpu_switcheroo_set_state(struct pci_dev *pdev,
1733                                         enum vga_switcheroo_state state)
1734 {
1735         struct drm_device *dev = pci_get_drvdata(pdev);
1736         int r;
1737
1738         if (amdgpu_device_supports_px(dev) && state == VGA_SWITCHEROO_OFF)
1739                 return;
1740
1741         if (state == VGA_SWITCHEROO_ON) {
1742                 pr_info("switched on\n");
1743                 /* don't suspend or resume card normally */
1744                 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1745
1746                 pci_set_power_state(pdev, PCI_D0);
1747                 amdgpu_device_load_pci_state(pdev);
1748                 r = pci_enable_device(pdev);
1749                 if (r)
1750                         DRM_WARN("pci_enable_device failed (%d)\n", r);
1751                 amdgpu_device_resume(dev, true);
1752
1753                 dev->switch_power_state = DRM_SWITCH_POWER_ON;
1754         } else {
1755                 pr_info("switched off\n");
1756                 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1757                 amdgpu_device_prepare(dev);
1758                 amdgpu_device_suspend(dev, true);
1759                 amdgpu_device_cache_pci_state(pdev);
1760                 /* Shut down the device */
1761                 pci_disable_device(pdev);
1762                 pci_set_power_state(pdev, PCI_D3cold);
1763                 dev->switch_power_state = DRM_SWITCH_POWER_OFF;
1764         }
1765 }
1766
1767 /**
1768  * amdgpu_switcheroo_can_switch - see if switcheroo state can change
1769  *
1770  * @pdev: pci dev pointer
1771  *
1772  * Callback for the switcheroo driver.  Check of the switcheroo
1773  * state can be changed.
1774  * Returns true if the state can be changed, false if not.
1775  */
1776 static bool amdgpu_switcheroo_can_switch(struct pci_dev *pdev)
1777 {
1778         struct drm_device *dev = pci_get_drvdata(pdev);
1779
1780        /*
1781         * FIXME: open_count is protected by drm_global_mutex but that would lead to
1782         * locking inversion with the driver load path. And the access here is
1783         * completely racy anyway. So don't bother with locking for now.
1784         */
1785         return atomic_read(&dev->open_count) == 0;
1786 }
1787
1788 static const struct vga_switcheroo_client_ops amdgpu_switcheroo_ops = {
1789         .set_gpu_state = amdgpu_switcheroo_set_state,
1790         .reprobe = NULL,
1791         .can_switch = amdgpu_switcheroo_can_switch,
1792 };
1793
1794 /**
1795  * amdgpu_device_ip_set_clockgating_state - set the CG state
1796  *
1797  * @dev: amdgpu_device pointer
1798  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1799  * @state: clockgating state (gate or ungate)
1800  *
1801  * Sets the requested clockgating state for all instances of
1802  * the hardware IP specified.
1803  * Returns the error code from the last instance.
1804  */
1805 int amdgpu_device_ip_set_clockgating_state(void *dev,
1806                                            enum amd_ip_block_type block_type,
1807                                            enum amd_clockgating_state state)
1808 {
1809         struct amdgpu_device *adev = dev;
1810         int i, r = 0;
1811
1812         for (i = 0; i < adev->num_ip_blocks; i++) {
1813                 if (!adev->ip_blocks[i].status.valid)
1814                         continue;
1815                 if (adev->ip_blocks[i].version->type != block_type)
1816                         continue;
1817                 if (!adev->ip_blocks[i].version->funcs->set_clockgating_state)
1818                         continue;
1819                 r = adev->ip_blocks[i].version->funcs->set_clockgating_state(
1820                         (void *)adev, state);
1821                 if (r)
1822                         DRM_ERROR("set_clockgating_state of IP block <%s> failed %d\n",
1823                                   adev->ip_blocks[i].version->funcs->name, r);
1824         }
1825         return r;
1826 }
1827
1828 /**
1829  * amdgpu_device_ip_set_powergating_state - set the PG state
1830  *
1831  * @dev: amdgpu_device pointer
1832  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1833  * @state: powergating state (gate or ungate)
1834  *
1835  * Sets the requested powergating state for all instances of
1836  * the hardware IP specified.
1837  * Returns the error code from the last instance.
1838  */
1839 int amdgpu_device_ip_set_powergating_state(void *dev,
1840                                            enum amd_ip_block_type block_type,
1841                                            enum amd_powergating_state state)
1842 {
1843         struct amdgpu_device *adev = dev;
1844         int i, r = 0;
1845
1846         for (i = 0; i < adev->num_ip_blocks; i++) {
1847                 if (!adev->ip_blocks[i].status.valid)
1848                         continue;
1849                 if (adev->ip_blocks[i].version->type != block_type)
1850                         continue;
1851                 if (!adev->ip_blocks[i].version->funcs->set_powergating_state)
1852                         continue;
1853                 r = adev->ip_blocks[i].version->funcs->set_powergating_state(
1854                         (void *)adev, state);
1855                 if (r)
1856                         DRM_ERROR("set_powergating_state of IP block <%s> failed %d\n",
1857                                   adev->ip_blocks[i].version->funcs->name, r);
1858         }
1859         return r;
1860 }
1861
1862 /**
1863  * amdgpu_device_ip_get_clockgating_state - get the CG state
1864  *
1865  * @adev: amdgpu_device pointer
1866  * @flags: clockgating feature flags
1867  *
1868  * Walks the list of IPs on the device and updates the clockgating
1869  * flags for each IP.
1870  * Updates @flags with the feature flags for each hardware IP where
1871  * clockgating is enabled.
1872  */
1873 void amdgpu_device_ip_get_clockgating_state(struct amdgpu_device *adev,
1874                                             u64 *flags)
1875 {
1876         int i;
1877
1878         for (i = 0; i < adev->num_ip_blocks; i++) {
1879                 if (!adev->ip_blocks[i].status.valid)
1880                         continue;
1881                 if (adev->ip_blocks[i].version->funcs->get_clockgating_state)
1882                         adev->ip_blocks[i].version->funcs->get_clockgating_state((void *)adev, flags);
1883         }
1884 }
1885
1886 /**
1887  * amdgpu_device_ip_wait_for_idle - wait for idle
1888  *
1889  * @adev: amdgpu_device pointer
1890  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1891  *
1892  * Waits for the request hardware IP to be idle.
1893  * Returns 0 for success or a negative error code on failure.
1894  */
1895 int amdgpu_device_ip_wait_for_idle(struct amdgpu_device *adev,
1896                                    enum amd_ip_block_type block_type)
1897 {
1898         int i, r;
1899
1900         for (i = 0; i < adev->num_ip_blocks; i++) {
1901                 if (!adev->ip_blocks[i].status.valid)
1902                         continue;
1903                 if (adev->ip_blocks[i].version->type == block_type) {
1904                         r = adev->ip_blocks[i].version->funcs->wait_for_idle((void *)adev);
1905                         if (r)
1906                                 return r;
1907                         break;
1908                 }
1909         }
1910         return 0;
1911
1912 }
1913
1914 /**
1915  * amdgpu_device_ip_is_idle - is the hardware IP idle
1916  *
1917  * @adev: amdgpu_device pointer
1918  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1919  *
1920  * Check if the hardware IP is idle or not.
1921  * Returns true if it the IP is idle, false if not.
1922  */
1923 bool amdgpu_device_ip_is_idle(struct amdgpu_device *adev,
1924                               enum amd_ip_block_type block_type)
1925 {
1926         int i;
1927
1928         for (i = 0; i < adev->num_ip_blocks; i++) {
1929                 if (!adev->ip_blocks[i].status.valid)
1930                         continue;
1931                 if (adev->ip_blocks[i].version->type == block_type)
1932                         return adev->ip_blocks[i].version->funcs->is_idle((void *)adev);
1933         }
1934         return true;
1935
1936 }
1937
1938 /**
1939  * amdgpu_device_ip_get_ip_block - get a hw IP pointer
1940  *
1941  * @adev: amdgpu_device pointer
1942  * @type: Type of hardware IP (SMU, GFX, UVD, etc.)
1943  *
1944  * Returns a pointer to the hardware IP block structure
1945  * if it exists for the asic, otherwise NULL.
1946  */
1947 struct amdgpu_ip_block *
1948 amdgpu_device_ip_get_ip_block(struct amdgpu_device *adev,
1949                               enum amd_ip_block_type type)
1950 {
1951         int i;
1952
1953         for (i = 0; i < adev->num_ip_blocks; i++)
1954                 if (adev->ip_blocks[i].version->type == type)
1955                         return &adev->ip_blocks[i];
1956
1957         return NULL;
1958 }
1959
1960 /**
1961  * amdgpu_device_ip_block_version_cmp
1962  *
1963  * @adev: amdgpu_device pointer
1964  * @type: enum amd_ip_block_type
1965  * @major: major version
1966  * @minor: minor version
1967  *
1968  * return 0 if equal or greater
1969  * return 1 if smaller or the ip_block doesn't exist
1970  */
1971 int amdgpu_device_ip_block_version_cmp(struct amdgpu_device *adev,
1972                                        enum amd_ip_block_type type,
1973                                        u32 major, u32 minor)
1974 {
1975         struct amdgpu_ip_block *ip_block = amdgpu_device_ip_get_ip_block(adev, type);
1976
1977         if (ip_block && ((ip_block->version->major > major) ||
1978                         ((ip_block->version->major == major) &&
1979                         (ip_block->version->minor >= minor))))
1980                 return 0;
1981
1982         return 1;
1983 }
1984
1985 /**
1986  * amdgpu_device_ip_block_add
1987  *
1988  * @adev: amdgpu_device pointer
1989  * @ip_block_version: pointer to the IP to add
1990  *
1991  * Adds the IP block driver information to the collection of IPs
1992  * on the asic.
1993  */
1994 int amdgpu_device_ip_block_add(struct amdgpu_device *adev,
1995                                const struct amdgpu_ip_block_version *ip_block_version)
1996 {
1997         if (!ip_block_version)
1998                 return -EINVAL;
1999
2000         switch (ip_block_version->type) {
2001         case AMD_IP_BLOCK_TYPE_VCN:
2002                 if (adev->harvest_ip_mask & AMD_HARVEST_IP_VCN_MASK)
2003                         return 0;
2004                 break;
2005         case AMD_IP_BLOCK_TYPE_JPEG:
2006                 if (adev->harvest_ip_mask & AMD_HARVEST_IP_JPEG_MASK)
2007                         return 0;
2008                 break;
2009         default:
2010                 break;
2011         }
2012
2013         DRM_INFO("add ip block number %d <%s>\n", adev->num_ip_blocks,
2014                   ip_block_version->funcs->name);
2015
2016         adev->ip_blocks[adev->num_ip_blocks++].version = ip_block_version;
2017
2018         return 0;
2019 }
2020
2021 /**
2022  * amdgpu_device_enable_virtual_display - enable virtual display feature
2023  *
2024  * @adev: amdgpu_device pointer
2025  *
2026  * Enabled the virtual display feature if the user has enabled it via
2027  * the module parameter virtual_display.  This feature provides a virtual
2028  * display hardware on headless boards or in virtualized environments.
2029  * This function parses and validates the configuration string specified by
2030  * the user and configues the virtual display configuration (number of
2031  * virtual connectors, crtcs, etc.) specified.
2032  */
2033 static void amdgpu_device_enable_virtual_display(struct amdgpu_device *adev)
2034 {
2035         adev->enable_virtual_display = false;
2036
2037         if (amdgpu_virtual_display) {
2038                 const char *pci_address_name = pci_name(adev->pdev);
2039                 char *pciaddstr, *pciaddstr_tmp, *pciaddname_tmp, *pciaddname;
2040
2041                 pciaddstr = kstrdup(amdgpu_virtual_display, GFP_KERNEL);
2042                 pciaddstr_tmp = pciaddstr;
2043                 while ((pciaddname_tmp = strsep(&pciaddstr_tmp, ";"))) {
2044                         pciaddname = strsep(&pciaddname_tmp, ",");
2045                         if (!strcmp("all", pciaddname)
2046                             || !strcmp(pci_address_name, pciaddname)) {
2047                                 long num_crtc;
2048                                 int res = -1;
2049
2050                                 adev->enable_virtual_display = true;
2051
2052                                 if (pciaddname_tmp)
2053                                         res = kstrtol(pciaddname_tmp, 10,
2054                                                       &num_crtc);
2055
2056                                 if (!res) {
2057                                         if (num_crtc < 1)
2058                                                 num_crtc = 1;
2059                                         if (num_crtc > 6)
2060                                                 num_crtc = 6;
2061                                         adev->mode_info.num_crtc = num_crtc;
2062                                 } else {
2063                                         adev->mode_info.num_crtc = 1;
2064                                 }
2065                                 break;
2066                         }
2067                 }
2068
2069                 DRM_INFO("virtual display string:%s, %s:virtual_display:%d, num_crtc:%d\n",
2070                          amdgpu_virtual_display, pci_address_name,
2071                          adev->enable_virtual_display, adev->mode_info.num_crtc);
2072
2073                 kfree(pciaddstr);
2074         }
2075 }
2076
2077 void amdgpu_device_set_sriov_virtual_display(struct amdgpu_device *adev)
2078 {
2079         if (amdgpu_sriov_vf(adev) && !adev->enable_virtual_display) {
2080                 adev->mode_info.num_crtc = 1;
2081                 adev->enable_virtual_display = true;
2082                 DRM_INFO("virtual_display:%d, num_crtc:%d\n",
2083                          adev->enable_virtual_display, adev->mode_info.num_crtc);
2084         }
2085 }
2086
2087 /**
2088  * amdgpu_device_parse_gpu_info_fw - parse gpu info firmware
2089  *
2090  * @adev: amdgpu_device pointer
2091  *
2092  * Parses the asic configuration parameters specified in the gpu info
2093  * firmware and makes them availale to the driver for use in configuring
2094  * the asic.
2095  * Returns 0 on success, -EINVAL on failure.
2096  */
2097 static int amdgpu_device_parse_gpu_info_fw(struct amdgpu_device *adev)
2098 {
2099         const char *chip_name;
2100         char fw_name[40];
2101         int err;
2102         const struct gpu_info_firmware_header_v1_0 *hdr;
2103
2104         adev->firmware.gpu_info_fw = NULL;
2105
2106         if (adev->mman.discovery_bin) {
2107                 /*
2108                  * FIXME: The bounding box is still needed by Navi12, so
2109                  * temporarily read it from gpu_info firmware. Should be dropped
2110                  * when DAL no longer needs it.
2111                  */
2112                 if (adev->asic_type != CHIP_NAVI12)
2113                         return 0;
2114         }
2115
2116         switch (adev->asic_type) {
2117         default:
2118                 return 0;
2119         case CHIP_VEGA10:
2120                 chip_name = "vega10";
2121                 break;
2122         case CHIP_VEGA12:
2123                 chip_name = "vega12";
2124                 break;
2125         case CHIP_RAVEN:
2126                 if (adev->apu_flags & AMD_APU_IS_RAVEN2)
2127                         chip_name = "raven2";
2128                 else if (adev->apu_flags & AMD_APU_IS_PICASSO)
2129                         chip_name = "picasso";
2130                 else
2131                         chip_name = "raven";
2132                 break;
2133         case CHIP_ARCTURUS:
2134                 chip_name = "arcturus";
2135                 break;
2136         case CHIP_NAVI12:
2137                 chip_name = "navi12";
2138                 break;
2139         }
2140
2141         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_gpu_info.bin", chip_name);
2142         err = amdgpu_ucode_request(adev, &adev->firmware.gpu_info_fw, fw_name);
2143         if (err) {
2144                 dev_err(adev->dev,
2145                         "Failed to get gpu_info firmware \"%s\"\n",
2146                         fw_name);
2147                 goto out;
2148         }
2149
2150         hdr = (const struct gpu_info_firmware_header_v1_0 *)adev->firmware.gpu_info_fw->data;
2151         amdgpu_ucode_print_gpu_info_hdr(&hdr->header);
2152
2153         switch (hdr->version_major) {
2154         case 1:
2155         {
2156                 const struct gpu_info_firmware_v1_0 *gpu_info_fw =
2157                         (const struct gpu_info_firmware_v1_0 *)(adev->firmware.gpu_info_fw->data +
2158                                                                 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2159
2160                 /*
2161                  * Should be droped when DAL no longer needs it.
2162                  */
2163                 if (adev->asic_type == CHIP_NAVI12)
2164                         goto parse_soc_bounding_box;
2165
2166                 adev->gfx.config.max_shader_engines = le32_to_cpu(gpu_info_fw->gc_num_se);
2167                 adev->gfx.config.max_cu_per_sh = le32_to_cpu(gpu_info_fw->gc_num_cu_per_sh);
2168                 adev->gfx.config.max_sh_per_se = le32_to_cpu(gpu_info_fw->gc_num_sh_per_se);
2169                 adev->gfx.config.max_backends_per_se = le32_to_cpu(gpu_info_fw->gc_num_rb_per_se);
2170                 adev->gfx.config.max_texture_channel_caches =
2171                         le32_to_cpu(gpu_info_fw->gc_num_tccs);
2172                 adev->gfx.config.max_gprs = le32_to_cpu(gpu_info_fw->gc_num_gprs);
2173                 adev->gfx.config.max_gs_threads = le32_to_cpu(gpu_info_fw->gc_num_max_gs_thds);
2174                 adev->gfx.config.gs_vgt_table_depth = le32_to_cpu(gpu_info_fw->gc_gs_table_depth);
2175                 adev->gfx.config.gs_prim_buffer_depth = le32_to_cpu(gpu_info_fw->gc_gsprim_buff_depth);
2176                 adev->gfx.config.double_offchip_lds_buf =
2177                         le32_to_cpu(gpu_info_fw->gc_double_offchip_lds_buffer);
2178                 adev->gfx.cu_info.wave_front_size = le32_to_cpu(gpu_info_fw->gc_wave_size);
2179                 adev->gfx.cu_info.max_waves_per_simd =
2180                         le32_to_cpu(gpu_info_fw->gc_max_waves_per_simd);
2181                 adev->gfx.cu_info.max_scratch_slots_per_cu =
2182                         le32_to_cpu(gpu_info_fw->gc_max_scratch_slots_per_cu);
2183                 adev->gfx.cu_info.lds_size = le32_to_cpu(gpu_info_fw->gc_lds_size);
2184                 if (hdr->version_minor >= 1) {
2185                         const struct gpu_info_firmware_v1_1 *gpu_info_fw =
2186                                 (const struct gpu_info_firmware_v1_1 *)(adev->firmware.gpu_info_fw->data +
2187                                                                         le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2188                         adev->gfx.config.num_sc_per_sh =
2189                                 le32_to_cpu(gpu_info_fw->num_sc_per_sh);
2190                         adev->gfx.config.num_packer_per_sc =
2191                                 le32_to_cpu(gpu_info_fw->num_packer_per_sc);
2192                 }
2193
2194 parse_soc_bounding_box:
2195                 /*
2196                  * soc bounding box info is not integrated in disocovery table,
2197                  * we always need to parse it from gpu info firmware if needed.
2198                  */
2199                 if (hdr->version_minor == 2) {
2200                         const struct gpu_info_firmware_v1_2 *gpu_info_fw =
2201                                 (const struct gpu_info_firmware_v1_2 *)(adev->firmware.gpu_info_fw->data +
2202                                                                         le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2203                         adev->dm.soc_bounding_box = &gpu_info_fw->soc_bounding_box;
2204                 }
2205                 break;
2206         }
2207         default:
2208                 dev_err(adev->dev,
2209                         "Unsupported gpu_info table %d\n", hdr->header.ucode_version);
2210                 err = -EINVAL;
2211                 goto out;
2212         }
2213 out:
2214         return err;
2215 }
2216
2217 /**
2218  * amdgpu_device_ip_early_init - run early init for hardware IPs
2219  *
2220  * @adev: amdgpu_device pointer
2221  *
2222  * Early initialization pass for hardware IPs.  The hardware IPs that make
2223  * up each asic are discovered each IP's early_init callback is run.  This
2224  * is the first stage in initializing the asic.
2225  * Returns 0 on success, negative error code on failure.
2226  */
2227 static int amdgpu_device_ip_early_init(struct amdgpu_device *adev)
2228 {
2229         struct pci_dev *parent;
2230         int i, r;
2231         bool total;
2232
2233         amdgpu_device_enable_virtual_display(adev);
2234
2235         if (amdgpu_sriov_vf(adev)) {
2236                 r = amdgpu_virt_request_full_gpu(adev, true);
2237                 if (r)
2238                         return r;
2239         }
2240
2241         switch (adev->asic_type) {
2242 #ifdef CONFIG_DRM_AMDGPU_SI
2243         case CHIP_VERDE:
2244         case CHIP_TAHITI:
2245         case CHIP_PITCAIRN:
2246         case CHIP_OLAND:
2247         case CHIP_HAINAN:
2248                 adev->family = AMDGPU_FAMILY_SI;
2249                 r = si_set_ip_blocks(adev);
2250                 if (r)
2251                         return r;
2252                 break;
2253 #endif
2254 #ifdef CONFIG_DRM_AMDGPU_CIK
2255         case CHIP_BONAIRE:
2256         case CHIP_HAWAII:
2257         case CHIP_KAVERI:
2258         case CHIP_KABINI:
2259         case CHIP_MULLINS:
2260                 if (adev->flags & AMD_IS_APU)
2261                         adev->family = AMDGPU_FAMILY_KV;
2262                 else
2263                         adev->family = AMDGPU_FAMILY_CI;
2264
2265                 r = cik_set_ip_blocks(adev);
2266                 if (r)
2267                         return r;
2268                 break;
2269 #endif
2270         case CHIP_TOPAZ:
2271         case CHIP_TONGA:
2272         case CHIP_FIJI:
2273         case CHIP_POLARIS10:
2274         case CHIP_POLARIS11:
2275         case CHIP_POLARIS12:
2276         case CHIP_VEGAM:
2277         case CHIP_CARRIZO:
2278         case CHIP_STONEY:
2279                 if (adev->flags & AMD_IS_APU)
2280                         adev->family = AMDGPU_FAMILY_CZ;
2281                 else
2282                         adev->family = AMDGPU_FAMILY_VI;
2283
2284                 r = vi_set_ip_blocks(adev);
2285                 if (r)
2286                         return r;
2287                 break;
2288         default:
2289                 r = amdgpu_discovery_set_ip_blocks(adev);
2290                 if (r)
2291                         return r;
2292                 break;
2293         }
2294
2295         if (amdgpu_has_atpx() &&
2296             (amdgpu_is_atpx_hybrid() ||
2297              amdgpu_has_atpx_dgpu_power_cntl()) &&
2298             ((adev->flags & AMD_IS_APU) == 0) &&
2299             !dev_is_removable(&adev->pdev->dev))
2300                 adev->flags |= AMD_IS_PX;
2301
2302         if (!(adev->flags & AMD_IS_APU)) {
2303                 parent = pcie_find_root_port(adev->pdev);
2304                 adev->has_pr3 = parent ? pci_pr3_present(parent) : false;
2305         }
2306
2307
2308         adev->pm.pp_feature = amdgpu_pp_feature_mask;
2309         if (amdgpu_sriov_vf(adev) || sched_policy == KFD_SCHED_POLICY_NO_HWS)
2310                 adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
2311         if (amdgpu_sriov_vf(adev) && adev->asic_type == CHIP_SIENNA_CICHLID)
2312                 adev->pm.pp_feature &= ~PP_OVERDRIVE_MASK;
2313         if (!amdgpu_device_pcie_dynamic_switching_supported())
2314                 adev->pm.pp_feature &= ~PP_PCIE_DPM_MASK;
2315
2316         total = true;
2317         for (i = 0; i < adev->num_ip_blocks; i++) {
2318                 if ((amdgpu_ip_block_mask & (1 << i)) == 0) {
2319                         DRM_WARN("disabled ip block: %d <%s>\n",
2320                                   i, adev->ip_blocks[i].version->funcs->name);
2321                         adev->ip_blocks[i].status.valid = false;
2322                 } else {
2323                         if (adev->ip_blocks[i].version->funcs->early_init) {
2324                                 r = adev->ip_blocks[i].version->funcs->early_init((void *)adev);
2325                                 if (r == -ENOENT) {
2326                                         adev->ip_blocks[i].status.valid = false;
2327                                 } else if (r) {
2328                                         DRM_ERROR("early_init of IP block <%s> failed %d\n",
2329                                                   adev->ip_blocks[i].version->funcs->name, r);
2330                                         total = false;
2331                                 } else {
2332                                         adev->ip_blocks[i].status.valid = true;
2333                                 }
2334                         } else {
2335                                 adev->ip_blocks[i].status.valid = true;
2336                         }
2337                 }
2338                 /* get the vbios after the asic_funcs are set up */
2339                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) {
2340                         r = amdgpu_device_parse_gpu_info_fw(adev);
2341                         if (r)
2342                                 return r;
2343
2344                         /* Read BIOS */
2345                         if (amdgpu_device_read_bios(adev)) {
2346                                 if (!amdgpu_get_bios(adev))
2347                                         return -EINVAL;
2348
2349                                 r = amdgpu_atombios_init(adev);
2350                                 if (r) {
2351                                         dev_err(adev->dev, "amdgpu_atombios_init failed\n");
2352                                         amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_INIT_FAIL, 0, 0);
2353                                         return r;
2354                                 }
2355                         }
2356
2357                         /*get pf2vf msg info at it's earliest time*/
2358                         if (amdgpu_sriov_vf(adev))
2359                                 amdgpu_virt_init_data_exchange(adev);
2360
2361                 }
2362         }
2363         if (!total)
2364                 return -ENODEV;
2365
2366         amdgpu_amdkfd_device_probe(adev);
2367         adev->cg_flags &= amdgpu_cg_mask;
2368         adev->pg_flags &= amdgpu_pg_mask;
2369
2370         return 0;
2371 }
2372
2373 static int amdgpu_device_ip_hw_init_phase1(struct amdgpu_device *adev)
2374 {
2375         int i, r;
2376
2377         for (i = 0; i < adev->num_ip_blocks; i++) {
2378                 if (!adev->ip_blocks[i].status.sw)
2379                         continue;
2380                 if (adev->ip_blocks[i].status.hw)
2381                         continue;
2382                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
2383                     (amdgpu_sriov_vf(adev) && (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)) ||
2384                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) {
2385                         r = adev->ip_blocks[i].version->funcs->hw_init(adev);
2386                         if (r) {
2387                                 DRM_ERROR("hw_init of IP block <%s> failed %d\n",
2388                                           adev->ip_blocks[i].version->funcs->name, r);
2389                                 return r;
2390                         }
2391                         adev->ip_blocks[i].status.hw = true;
2392                 }
2393         }
2394
2395         return 0;
2396 }
2397
2398 static int amdgpu_device_ip_hw_init_phase2(struct amdgpu_device *adev)
2399 {
2400         int i, r;
2401
2402         for (i = 0; i < adev->num_ip_blocks; i++) {
2403                 if (!adev->ip_blocks[i].status.sw)
2404                         continue;
2405                 if (adev->ip_blocks[i].status.hw)
2406                         continue;
2407                 r = adev->ip_blocks[i].version->funcs->hw_init(adev);
2408                 if (r) {
2409                         DRM_ERROR("hw_init of IP block <%s> failed %d\n",
2410                                   adev->ip_blocks[i].version->funcs->name, r);
2411                         return r;
2412                 }
2413                 adev->ip_blocks[i].status.hw = true;
2414         }
2415
2416         return 0;
2417 }
2418
2419 static int amdgpu_device_fw_loading(struct amdgpu_device *adev)
2420 {
2421         int r = 0;
2422         int i;
2423         uint32_t smu_version;
2424
2425         if (adev->asic_type >= CHIP_VEGA10) {
2426                 for (i = 0; i < adev->num_ip_blocks; i++) {
2427                         if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_PSP)
2428                                 continue;
2429
2430                         if (!adev->ip_blocks[i].status.sw)
2431                                 continue;
2432
2433                         /* no need to do the fw loading again if already done*/
2434                         if (adev->ip_blocks[i].status.hw == true)
2435                                 break;
2436
2437                         if (amdgpu_in_reset(adev) || adev->in_suspend) {
2438                                 r = adev->ip_blocks[i].version->funcs->resume(adev);
2439                                 if (r) {
2440                                         DRM_ERROR("resume of IP block <%s> failed %d\n",
2441                                                           adev->ip_blocks[i].version->funcs->name, r);
2442                                         return r;
2443                                 }
2444                         } else {
2445                                 r = adev->ip_blocks[i].version->funcs->hw_init(adev);
2446                                 if (r) {
2447                                         DRM_ERROR("hw_init of IP block <%s> failed %d\n",
2448                                                           adev->ip_blocks[i].version->funcs->name, r);
2449                                         return r;
2450                                 }
2451                         }
2452
2453                         adev->ip_blocks[i].status.hw = true;
2454                         break;
2455                 }
2456         }
2457
2458         if (!amdgpu_sriov_vf(adev) || adev->asic_type == CHIP_TONGA)
2459                 r = amdgpu_pm_load_smu_firmware(adev, &smu_version);
2460
2461         return r;
2462 }
2463
2464 static int amdgpu_device_init_schedulers(struct amdgpu_device *adev)
2465 {
2466         long timeout;
2467         int r, i;
2468
2469         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
2470                 struct amdgpu_ring *ring = adev->rings[i];
2471
2472                 /* No need to setup the GPU scheduler for rings that don't need it */
2473                 if (!ring || ring->no_scheduler)
2474                         continue;
2475
2476                 switch (ring->funcs->type) {
2477                 case AMDGPU_RING_TYPE_GFX:
2478                         timeout = adev->gfx_timeout;
2479                         break;
2480                 case AMDGPU_RING_TYPE_COMPUTE:
2481                         timeout = adev->compute_timeout;
2482                         break;
2483                 case AMDGPU_RING_TYPE_SDMA:
2484                         timeout = adev->sdma_timeout;
2485                         break;
2486                 default:
2487                         timeout = adev->video_timeout;
2488                         break;
2489                 }
2490
2491                 r = drm_sched_init(&ring->sched, &amdgpu_sched_ops,
2492                                    DRM_SCHED_PRIORITY_COUNT,
2493                                    ring->num_hw_submission, 0,
2494                                    timeout, adev->reset_domain->wq,
2495                                    ring->sched_score, ring->name,
2496                                    adev->dev);
2497                 if (r) {
2498                         DRM_ERROR("Failed to create scheduler on ring %s.\n",
2499                                   ring->name);
2500                         return r;
2501                 }
2502         }
2503
2504         amdgpu_xcp_update_partition_sched_list(adev);
2505
2506         return 0;
2507 }
2508
2509
2510 /**
2511  * amdgpu_device_ip_init - run init for hardware IPs
2512  *
2513  * @adev: amdgpu_device pointer
2514  *
2515  * Main initialization pass for hardware IPs.  The list of all the hardware
2516  * IPs that make up the asic is walked and the sw_init and hw_init callbacks
2517  * are run.  sw_init initializes the software state associated with each IP
2518  * and hw_init initializes the hardware associated with each IP.
2519  * Returns 0 on success, negative error code on failure.
2520  */
2521 static int amdgpu_device_ip_init(struct amdgpu_device *adev)
2522 {
2523         int i, r;
2524
2525         r = amdgpu_ras_init(adev);
2526         if (r)
2527                 return r;
2528
2529         for (i = 0; i < adev->num_ip_blocks; i++) {
2530                 if (!adev->ip_blocks[i].status.valid)
2531                         continue;
2532                 r = adev->ip_blocks[i].version->funcs->sw_init((void *)adev);
2533                 if (r) {
2534                         DRM_ERROR("sw_init of IP block <%s> failed %d\n",
2535                                   adev->ip_blocks[i].version->funcs->name, r);
2536                         goto init_failed;
2537                 }
2538                 adev->ip_blocks[i].status.sw = true;
2539
2540                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) {
2541                         /* need to do common hw init early so everything is set up for gmc */
2542                         r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev);
2543                         if (r) {
2544                                 DRM_ERROR("hw_init %d failed %d\n", i, r);
2545                                 goto init_failed;
2546                         }
2547                         adev->ip_blocks[i].status.hw = true;
2548                 } else if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
2549                         /* need to do gmc hw init early so we can allocate gpu mem */
2550                         /* Try to reserve bad pages early */
2551                         if (amdgpu_sriov_vf(adev))
2552                                 amdgpu_virt_exchange_data(adev);
2553
2554                         r = amdgpu_device_mem_scratch_init(adev);
2555                         if (r) {
2556                                 DRM_ERROR("amdgpu_mem_scratch_init failed %d\n", r);
2557                                 goto init_failed;
2558                         }
2559                         r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev);
2560                         if (r) {
2561                                 DRM_ERROR("hw_init %d failed %d\n", i, r);
2562                                 goto init_failed;
2563                         }
2564                         r = amdgpu_device_wb_init(adev);
2565                         if (r) {
2566                                 DRM_ERROR("amdgpu_device_wb_init failed %d\n", r);
2567                                 goto init_failed;
2568                         }
2569                         adev->ip_blocks[i].status.hw = true;
2570
2571                         /* right after GMC hw init, we create CSA */
2572                         if (adev->gfx.mcbp) {
2573                                 r = amdgpu_allocate_static_csa(adev, &adev->virt.csa_obj,
2574                                                                AMDGPU_GEM_DOMAIN_VRAM |
2575                                                                AMDGPU_GEM_DOMAIN_GTT,
2576                                                                AMDGPU_CSA_SIZE);
2577                                 if (r) {
2578                                         DRM_ERROR("allocate CSA failed %d\n", r);
2579                                         goto init_failed;
2580                                 }
2581                         }
2582                 }
2583         }
2584
2585         if (amdgpu_sriov_vf(adev))
2586                 amdgpu_virt_init_data_exchange(adev);
2587
2588         r = amdgpu_ib_pool_init(adev);
2589         if (r) {
2590                 dev_err(adev->dev, "IB initialization failed (%d).\n", r);
2591                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_IB_INIT_FAIL, 0, r);
2592                 goto init_failed;
2593         }
2594
2595         r = amdgpu_ucode_create_bo(adev); /* create ucode bo when sw_init complete*/
2596         if (r)
2597                 goto init_failed;
2598
2599         r = amdgpu_device_ip_hw_init_phase1(adev);
2600         if (r)
2601                 goto init_failed;
2602
2603         r = amdgpu_device_fw_loading(adev);
2604         if (r)
2605                 goto init_failed;
2606
2607         r = amdgpu_device_ip_hw_init_phase2(adev);
2608         if (r)
2609                 goto init_failed;
2610
2611         /*
2612          * retired pages will be loaded from eeprom and reserved here,
2613          * it should be called after amdgpu_device_ip_hw_init_phase2  since
2614          * for some ASICs the RAS EEPROM code relies on SMU fully functioning
2615          * for I2C communication which only true at this point.
2616          *
2617          * amdgpu_ras_recovery_init may fail, but the upper only cares the
2618          * failure from bad gpu situation and stop amdgpu init process
2619          * accordingly. For other failed cases, it will still release all
2620          * the resource and print error message, rather than returning one
2621          * negative value to upper level.
2622          *
2623          * Note: theoretically, this should be called before all vram allocations
2624          * to protect retired page from abusing
2625          */
2626         r = amdgpu_ras_recovery_init(adev);
2627         if (r)
2628                 goto init_failed;
2629
2630         /**
2631          * In case of XGMI grab extra reference for reset domain for this device
2632          */
2633         if (adev->gmc.xgmi.num_physical_nodes > 1) {
2634                 if (amdgpu_xgmi_add_device(adev) == 0) {
2635                         if (!amdgpu_sriov_vf(adev)) {
2636                                 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
2637
2638                                 if (WARN_ON(!hive)) {
2639                                         r = -ENOENT;
2640                                         goto init_failed;
2641                                 }
2642
2643                                 if (!hive->reset_domain ||
2644                                     !amdgpu_reset_get_reset_domain(hive->reset_domain)) {
2645                                         r = -ENOENT;
2646                                         amdgpu_put_xgmi_hive(hive);
2647                                         goto init_failed;
2648                                 }
2649
2650                                 /* Drop the early temporary reset domain we created for device */
2651                                 amdgpu_reset_put_reset_domain(adev->reset_domain);
2652                                 adev->reset_domain = hive->reset_domain;
2653                                 amdgpu_put_xgmi_hive(hive);
2654                         }
2655                 }
2656         }
2657
2658         r = amdgpu_device_init_schedulers(adev);
2659         if (r)
2660                 goto init_failed;
2661
2662         if (adev->mman.buffer_funcs_ring->sched.ready)
2663                 amdgpu_ttm_set_buffer_funcs_status(adev, true);
2664
2665         /* Don't init kfd if whole hive need to be reset during init */
2666         if (!adev->gmc.xgmi.pending_reset) {
2667                 kgd2kfd_init_zone_device(adev);
2668                 amdgpu_amdkfd_device_init(adev);
2669         }
2670
2671         amdgpu_fru_get_product_info(adev);
2672
2673 init_failed:
2674
2675         return r;
2676 }
2677
2678 /**
2679  * amdgpu_device_fill_reset_magic - writes reset magic to gart pointer
2680  *
2681  * @adev: amdgpu_device pointer
2682  *
2683  * Writes a reset magic value to the gart pointer in VRAM.  The driver calls
2684  * this function before a GPU reset.  If the value is retained after a
2685  * GPU reset, VRAM has not been lost.  Some GPU resets may destry VRAM contents.
2686  */
2687 static void amdgpu_device_fill_reset_magic(struct amdgpu_device *adev)
2688 {
2689         memcpy(adev->reset_magic, adev->gart.ptr, AMDGPU_RESET_MAGIC_NUM);
2690 }
2691
2692 /**
2693  * amdgpu_device_check_vram_lost - check if vram is valid
2694  *
2695  * @adev: amdgpu_device pointer
2696  *
2697  * Checks the reset magic value written to the gart pointer in VRAM.
2698  * The driver calls this after a GPU reset to see if the contents of
2699  * VRAM is lost or now.
2700  * returns true if vram is lost, false if not.
2701  */
2702 static bool amdgpu_device_check_vram_lost(struct amdgpu_device *adev)
2703 {
2704         if (memcmp(adev->gart.ptr, adev->reset_magic,
2705                         AMDGPU_RESET_MAGIC_NUM))
2706                 return true;
2707
2708         if (!amdgpu_in_reset(adev))
2709                 return false;
2710
2711         /*
2712          * For all ASICs with baco/mode1 reset, the VRAM is
2713          * always assumed to be lost.
2714          */
2715         switch (amdgpu_asic_reset_method(adev)) {
2716         case AMD_RESET_METHOD_BACO:
2717         case AMD_RESET_METHOD_MODE1:
2718                 return true;
2719         default:
2720                 return false;
2721         }
2722 }
2723
2724 /**
2725  * amdgpu_device_set_cg_state - set clockgating for amdgpu device
2726  *
2727  * @adev: amdgpu_device pointer
2728  * @state: clockgating state (gate or ungate)
2729  *
2730  * The list of all the hardware IPs that make up the asic is walked and the
2731  * set_clockgating_state callbacks are run.
2732  * Late initialization pass enabling clockgating for hardware IPs.
2733  * Fini or suspend, pass disabling clockgating for hardware IPs.
2734  * Returns 0 on success, negative error code on failure.
2735  */
2736
2737 int amdgpu_device_set_cg_state(struct amdgpu_device *adev,
2738                                enum amd_clockgating_state state)
2739 {
2740         int i, j, r;
2741
2742         if (amdgpu_emu_mode == 1)
2743                 return 0;
2744
2745         for (j = 0; j < adev->num_ip_blocks; j++) {
2746                 i = state == AMD_CG_STATE_GATE ? j : adev->num_ip_blocks - j - 1;
2747                 if (!adev->ip_blocks[i].status.late_initialized)
2748                         continue;
2749                 /* skip CG for GFX, SDMA on S0ix */
2750                 if (adev->in_s0ix &&
2751                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX ||
2752                      adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SDMA))
2753                         continue;
2754                 /* skip CG for VCE/UVD, it's handled specially */
2755                 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
2756                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&
2757                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&
2758                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG &&
2759                     adev->ip_blocks[i].version->funcs->set_clockgating_state) {
2760                         /* enable clockgating to save power */
2761                         r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
2762                                                                                      state);
2763                         if (r) {
2764                                 DRM_ERROR("set_clockgating_state(gate) of IP block <%s> failed %d\n",
2765                                           adev->ip_blocks[i].version->funcs->name, r);
2766                                 return r;
2767                         }
2768                 }
2769         }
2770
2771         return 0;
2772 }
2773
2774 int amdgpu_device_set_pg_state(struct amdgpu_device *adev,
2775                                enum amd_powergating_state state)
2776 {
2777         int i, j, r;
2778
2779         if (amdgpu_emu_mode == 1)
2780                 return 0;
2781
2782         for (j = 0; j < adev->num_ip_blocks; j++) {
2783                 i = state == AMD_PG_STATE_GATE ? j : adev->num_ip_blocks - j - 1;
2784                 if (!adev->ip_blocks[i].status.late_initialized)
2785                         continue;
2786                 /* skip PG for GFX, SDMA on S0ix */
2787                 if (adev->in_s0ix &&
2788                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX ||
2789                      adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SDMA))
2790                         continue;
2791                 /* skip CG for VCE/UVD, it's handled specially */
2792                 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
2793                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&
2794                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&
2795                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG &&
2796                     adev->ip_blocks[i].version->funcs->set_powergating_state) {
2797                         /* enable powergating to save power */
2798                         r = adev->ip_blocks[i].version->funcs->set_powergating_state((void *)adev,
2799                                                                                         state);
2800                         if (r) {
2801                                 DRM_ERROR("set_powergating_state(gate) of IP block <%s> failed %d\n",
2802                                           adev->ip_blocks[i].version->funcs->name, r);
2803                                 return r;
2804                         }
2805                 }
2806         }
2807         return 0;
2808 }
2809
2810 static int amdgpu_device_enable_mgpu_fan_boost(void)
2811 {
2812         struct amdgpu_gpu_instance *gpu_ins;
2813         struct amdgpu_device *adev;
2814         int i, ret = 0;
2815
2816         mutex_lock(&mgpu_info.mutex);
2817
2818         /*
2819          * MGPU fan boost feature should be enabled
2820          * only when there are two or more dGPUs in
2821          * the system
2822          */
2823         if (mgpu_info.num_dgpu < 2)
2824                 goto out;
2825
2826         for (i = 0; i < mgpu_info.num_dgpu; i++) {
2827                 gpu_ins = &(mgpu_info.gpu_ins[i]);
2828                 adev = gpu_ins->adev;
2829                 if (!(adev->flags & AMD_IS_APU) &&
2830                     !gpu_ins->mgpu_fan_enabled) {
2831                         ret = amdgpu_dpm_enable_mgpu_fan_boost(adev);
2832                         if (ret)
2833                                 break;
2834
2835                         gpu_ins->mgpu_fan_enabled = 1;
2836                 }
2837         }
2838
2839 out:
2840         mutex_unlock(&mgpu_info.mutex);
2841
2842         return ret;
2843 }
2844
2845 /**
2846  * amdgpu_device_ip_late_init - run late init for hardware IPs
2847  *
2848  * @adev: amdgpu_device pointer
2849  *
2850  * Late initialization pass for hardware IPs.  The list of all the hardware
2851  * IPs that make up the asic is walked and the late_init callbacks are run.
2852  * late_init covers any special initialization that an IP requires
2853  * after all of the have been initialized or something that needs to happen
2854  * late in the init process.
2855  * Returns 0 on success, negative error code on failure.
2856  */
2857 static int amdgpu_device_ip_late_init(struct amdgpu_device *adev)
2858 {
2859         struct amdgpu_gpu_instance *gpu_instance;
2860         int i = 0, r;
2861
2862         for (i = 0; i < adev->num_ip_blocks; i++) {
2863                 if (!adev->ip_blocks[i].status.hw)
2864                         continue;
2865                 if (adev->ip_blocks[i].version->funcs->late_init) {
2866                         r = adev->ip_blocks[i].version->funcs->late_init((void *)adev);
2867                         if (r) {
2868                                 DRM_ERROR("late_init of IP block <%s> failed %d\n",
2869                                           adev->ip_blocks[i].version->funcs->name, r);
2870                                 return r;
2871                         }
2872                 }
2873                 adev->ip_blocks[i].status.late_initialized = true;
2874         }
2875
2876         r = amdgpu_ras_late_init(adev);
2877         if (r) {
2878                 DRM_ERROR("amdgpu_ras_late_init failed %d", r);
2879                 return r;
2880         }
2881
2882         amdgpu_ras_set_error_query_ready(adev, true);
2883
2884         amdgpu_device_set_cg_state(adev, AMD_CG_STATE_GATE);
2885         amdgpu_device_set_pg_state(adev, AMD_PG_STATE_GATE);
2886
2887         amdgpu_device_fill_reset_magic(adev);
2888
2889         r = amdgpu_device_enable_mgpu_fan_boost();
2890         if (r)
2891                 DRM_ERROR("enable mgpu fan boost failed (%d).\n", r);
2892
2893         /* For passthrough configuration on arcturus and aldebaran, enable special handling SBR */
2894         if (amdgpu_passthrough(adev) &&
2895             ((adev->asic_type == CHIP_ARCTURUS && adev->gmc.xgmi.num_physical_nodes > 1) ||
2896              adev->asic_type == CHIP_ALDEBARAN))
2897                 amdgpu_dpm_handle_passthrough_sbr(adev, true);
2898
2899         if (adev->gmc.xgmi.num_physical_nodes > 1) {
2900                 mutex_lock(&mgpu_info.mutex);
2901
2902                 /*
2903                  * Reset device p-state to low as this was booted with high.
2904                  *
2905                  * This should be performed only after all devices from the same
2906                  * hive get initialized.
2907                  *
2908                  * However, it's unknown how many device in the hive in advance.
2909                  * As this is counted one by one during devices initializations.
2910                  *
2911                  * So, we wait for all XGMI interlinked devices initialized.
2912                  * This may bring some delays as those devices may come from
2913                  * different hives. But that should be OK.
2914                  */
2915                 if (mgpu_info.num_dgpu == adev->gmc.xgmi.num_physical_nodes) {
2916                         for (i = 0; i < mgpu_info.num_gpu; i++) {
2917                                 gpu_instance = &(mgpu_info.gpu_ins[i]);
2918                                 if (gpu_instance->adev->flags & AMD_IS_APU)
2919                                         continue;
2920
2921                                 r = amdgpu_xgmi_set_pstate(gpu_instance->adev,
2922                                                 AMDGPU_XGMI_PSTATE_MIN);
2923                                 if (r) {
2924                                         DRM_ERROR("pstate setting failed (%d).\n", r);
2925                                         break;
2926                                 }
2927                         }
2928                 }
2929
2930                 mutex_unlock(&mgpu_info.mutex);
2931         }
2932
2933         return 0;
2934 }
2935
2936 /**
2937  * amdgpu_device_smu_fini_early - smu hw_fini wrapper
2938  *
2939  * @adev: amdgpu_device pointer
2940  *
2941  * For ASICs need to disable SMC first
2942  */
2943 static void amdgpu_device_smu_fini_early(struct amdgpu_device *adev)
2944 {
2945         int i, r;
2946
2947         if (amdgpu_ip_version(adev, GC_HWIP, 0) > IP_VERSION(9, 0, 0))
2948                 return;
2949
2950         for (i = 0; i < adev->num_ip_blocks; i++) {
2951                 if (!adev->ip_blocks[i].status.hw)
2952                         continue;
2953                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
2954                         r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
2955                         /* XXX handle errors */
2956                         if (r) {
2957                                 DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
2958                                           adev->ip_blocks[i].version->funcs->name, r);
2959                         }
2960                         adev->ip_blocks[i].status.hw = false;
2961                         break;
2962                 }
2963         }
2964 }
2965
2966 static int amdgpu_device_ip_fini_early(struct amdgpu_device *adev)
2967 {
2968         int i, r;
2969
2970         for (i = 0; i < adev->num_ip_blocks; i++) {
2971                 if (!adev->ip_blocks[i].version->funcs->early_fini)
2972                         continue;
2973
2974                 r = adev->ip_blocks[i].version->funcs->early_fini((void *)adev);
2975                 if (r) {
2976                         DRM_DEBUG("early_fini of IP block <%s> failed %d\n",
2977                                   adev->ip_blocks[i].version->funcs->name, r);
2978                 }
2979         }
2980
2981         amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE);
2982         amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE);
2983
2984         amdgpu_amdkfd_suspend(adev, false);
2985
2986         /* Workaroud for ASICs need to disable SMC first */
2987         amdgpu_device_smu_fini_early(adev);
2988
2989         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2990                 if (!adev->ip_blocks[i].status.hw)
2991                         continue;
2992
2993                 r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
2994                 /* XXX handle errors */
2995                 if (r) {
2996                         DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
2997                                   adev->ip_blocks[i].version->funcs->name, r);
2998                 }
2999
3000                 adev->ip_blocks[i].status.hw = false;
3001         }
3002
3003         if (amdgpu_sriov_vf(adev)) {
3004                 if (amdgpu_virt_release_full_gpu(adev, false))
3005                         DRM_ERROR("failed to release exclusive mode on fini\n");
3006         }
3007
3008         return 0;
3009 }
3010
3011 /**
3012  * amdgpu_device_ip_fini - run fini for hardware IPs
3013  *
3014  * @adev: amdgpu_device pointer
3015  *
3016  * Main teardown pass for hardware IPs.  The list of all the hardware
3017  * IPs that make up the asic is walked and the hw_fini and sw_fini callbacks
3018  * are run.  hw_fini tears down the hardware associated with each IP
3019  * and sw_fini tears down any software state associated with each IP.
3020  * Returns 0 on success, negative error code on failure.
3021  */
3022 static int amdgpu_device_ip_fini(struct amdgpu_device *adev)
3023 {
3024         int i, r;
3025
3026         if (amdgpu_sriov_vf(adev) && adev->virt.ras_init_done)
3027                 amdgpu_virt_release_ras_err_handler_data(adev);
3028
3029         if (adev->gmc.xgmi.num_physical_nodes > 1)
3030                 amdgpu_xgmi_remove_device(adev);
3031
3032         amdgpu_amdkfd_device_fini_sw(adev);
3033
3034         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
3035                 if (!adev->ip_blocks[i].status.sw)
3036                         continue;
3037
3038                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
3039                         amdgpu_ucode_free_bo(adev);
3040                         amdgpu_free_static_csa(&adev->virt.csa_obj);
3041                         amdgpu_device_wb_fini(adev);
3042                         amdgpu_device_mem_scratch_fini(adev);
3043                         amdgpu_ib_pool_fini(adev);
3044                 }
3045
3046                 r = adev->ip_blocks[i].version->funcs->sw_fini((void *)adev);
3047                 /* XXX handle errors */
3048                 if (r) {
3049                         DRM_DEBUG("sw_fini of IP block <%s> failed %d\n",
3050                                   adev->ip_blocks[i].version->funcs->name, r);
3051                 }
3052                 adev->ip_blocks[i].status.sw = false;
3053                 adev->ip_blocks[i].status.valid = false;
3054         }
3055
3056         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
3057                 if (!adev->ip_blocks[i].status.late_initialized)
3058                         continue;
3059                 if (adev->ip_blocks[i].version->funcs->late_fini)
3060                         adev->ip_blocks[i].version->funcs->late_fini((void *)adev);
3061                 adev->ip_blocks[i].status.late_initialized = false;
3062         }
3063
3064         amdgpu_ras_fini(adev);
3065
3066         return 0;
3067 }
3068
3069 /**
3070  * amdgpu_device_delayed_init_work_handler - work handler for IB tests
3071  *
3072  * @work: work_struct.
3073  */
3074 static void amdgpu_device_delayed_init_work_handler(struct work_struct *work)
3075 {
3076         struct amdgpu_device *adev =
3077                 container_of(work, struct amdgpu_device, delayed_init_work.work);
3078         int r;
3079
3080         r = amdgpu_ib_ring_tests(adev);
3081         if (r)
3082                 DRM_ERROR("ib ring test failed (%d).\n", r);
3083 }
3084
3085 static void amdgpu_device_delay_enable_gfx_off(struct work_struct *work)
3086 {
3087         struct amdgpu_device *adev =
3088                 container_of(work, struct amdgpu_device, gfx.gfx_off_delay_work.work);
3089
3090         WARN_ON_ONCE(adev->gfx.gfx_off_state);
3091         WARN_ON_ONCE(adev->gfx.gfx_off_req_count);
3092
3093         if (!amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, true))
3094                 adev->gfx.gfx_off_state = true;
3095 }
3096
3097 /**
3098  * amdgpu_device_ip_suspend_phase1 - run suspend for hardware IPs (phase 1)
3099  *
3100  * @adev: amdgpu_device pointer
3101  *
3102  * Main suspend function for hardware IPs.  The list of all the hardware
3103  * IPs that make up the asic is walked, clockgating is disabled and the
3104  * suspend callbacks are run.  suspend puts the hardware and software state
3105  * in each IP into a state suitable for suspend.
3106  * Returns 0 on success, negative error code on failure.
3107  */
3108 static int amdgpu_device_ip_suspend_phase1(struct amdgpu_device *adev)
3109 {
3110         int i, r;
3111
3112         amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE);
3113         amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE);
3114
3115         /*
3116          * Per PMFW team's suggestion, driver needs to handle gfxoff
3117          * and df cstate features disablement for gpu reset(e.g. Mode1Reset)
3118          * scenario. Add the missing df cstate disablement here.
3119          */
3120         if (amdgpu_dpm_set_df_cstate(adev, DF_CSTATE_DISALLOW))
3121                 dev_warn(adev->dev, "Failed to disallow df cstate");
3122
3123         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
3124                 if (!adev->ip_blocks[i].status.valid)
3125                         continue;
3126
3127                 /* displays are handled separately */
3128                 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_DCE)
3129                         continue;
3130
3131                 /* XXX handle errors */
3132                 r = adev->ip_blocks[i].version->funcs->suspend(adev);
3133                 /* XXX handle errors */
3134                 if (r) {
3135                         DRM_ERROR("suspend of IP block <%s> failed %d\n",
3136                                   adev->ip_blocks[i].version->funcs->name, r);
3137                         return r;
3138                 }
3139
3140                 adev->ip_blocks[i].status.hw = false;
3141         }
3142
3143         return 0;
3144 }
3145
3146 /**
3147  * amdgpu_device_ip_suspend_phase2 - run suspend for hardware IPs (phase 2)
3148  *
3149  * @adev: amdgpu_device pointer
3150  *
3151  * Main suspend function for hardware IPs.  The list of all the hardware
3152  * IPs that make up the asic is walked, clockgating is disabled and the
3153  * suspend callbacks are run.  suspend puts the hardware and software state
3154  * in each IP into a state suitable for suspend.
3155  * Returns 0 on success, negative error code on failure.
3156  */
3157 static int amdgpu_device_ip_suspend_phase2(struct amdgpu_device *adev)
3158 {
3159         int i, r;
3160
3161         if (adev->in_s0ix)
3162                 amdgpu_dpm_gfx_state_change(adev, sGpuChangeState_D3Entry);
3163
3164         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
3165                 if (!adev->ip_blocks[i].status.valid)
3166                         continue;
3167                 /* displays are handled in phase1 */
3168                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE)
3169                         continue;
3170                 /* PSP lost connection when err_event_athub occurs */
3171                 if (amdgpu_ras_intr_triggered() &&
3172                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) {
3173                         adev->ip_blocks[i].status.hw = false;
3174                         continue;
3175                 }
3176
3177                 /* skip unnecessary suspend if we do not initialize them yet */
3178                 if (adev->gmc.xgmi.pending_reset &&
3179                     !(adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3180                       adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC ||
3181                       adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3182                       adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH)) {
3183                         adev->ip_blocks[i].status.hw = false;
3184                         continue;
3185                 }
3186
3187                 /* skip suspend of gfx/mes and psp for S0ix
3188                  * gfx is in gfxoff state, so on resume it will exit gfxoff just
3189                  * like at runtime. PSP is also part of the always on hardware
3190                  * so no need to suspend it.
3191                  */
3192                 if (adev->in_s0ix &&
3193                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP ||
3194                      adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX ||
3195                      adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_MES))
3196                         continue;
3197
3198                 /* SDMA 5.x+ is part of GFX power domain so it's covered by GFXOFF */
3199                 if (adev->in_s0ix &&
3200                     (amdgpu_ip_version(adev, SDMA0_HWIP, 0) >=
3201                      IP_VERSION(5, 0, 0)) &&
3202                     (adev->ip_blocks[i].version->type ==
3203                      AMD_IP_BLOCK_TYPE_SDMA))
3204                         continue;
3205
3206                 /* Once swPSP provides the IMU, RLC FW binaries to TOS during cold-boot.
3207                  * These are in TMR, hence are expected to be reused by PSP-TOS to reload
3208                  * from this location and RLC Autoload automatically also gets loaded
3209                  * from here based on PMFW -> PSP message during re-init sequence.
3210                  * Therefore, the psp suspend & resume should be skipped to avoid destroy
3211                  * the TMR and reload FWs again for IMU enabled APU ASICs.
3212                  */
3213                 if (amdgpu_in_reset(adev) &&
3214                     (adev->flags & AMD_IS_APU) && adev->gfx.imu.funcs &&
3215                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)
3216                         continue;
3217
3218                 /* XXX handle errors */
3219                 r = adev->ip_blocks[i].version->funcs->suspend(adev);
3220                 /* XXX handle errors */
3221                 if (r) {
3222                         DRM_ERROR("suspend of IP block <%s> failed %d\n",
3223                                   adev->ip_blocks[i].version->funcs->name, r);
3224                 }
3225                 adev->ip_blocks[i].status.hw = false;
3226                 /* handle putting the SMC in the appropriate state */
3227                 if (!amdgpu_sriov_vf(adev)) {
3228                         if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
3229                                 r = amdgpu_dpm_set_mp1_state(adev, adev->mp1_state);
3230                                 if (r) {
3231                                         DRM_ERROR("SMC failed to set mp1 state %d, %d\n",
3232                                                         adev->mp1_state, r);
3233                                         return r;
3234                                 }
3235                         }
3236                 }
3237         }
3238
3239         return 0;
3240 }
3241
3242 /**
3243  * amdgpu_device_ip_suspend - run suspend for hardware IPs
3244  *
3245  * @adev: amdgpu_device pointer
3246  *
3247  * Main suspend function for hardware IPs.  The list of all the hardware
3248  * IPs that make up the asic is walked, clockgating is disabled and the
3249  * suspend callbacks are run.  suspend puts the hardware and software state
3250  * in each IP into a state suitable for suspend.
3251  * Returns 0 on success, negative error code on failure.
3252  */
3253 int amdgpu_device_ip_suspend(struct amdgpu_device *adev)
3254 {
3255         int r;
3256
3257         if (amdgpu_sriov_vf(adev)) {
3258                 amdgpu_virt_fini_data_exchange(adev);
3259                 amdgpu_virt_request_full_gpu(adev, false);
3260         }
3261
3262         amdgpu_ttm_set_buffer_funcs_status(adev, false);
3263
3264         r = amdgpu_device_ip_suspend_phase1(adev);
3265         if (r)
3266                 return r;
3267         r = amdgpu_device_ip_suspend_phase2(adev);
3268
3269         if (amdgpu_sriov_vf(adev))
3270                 amdgpu_virt_release_full_gpu(adev, false);
3271
3272         return r;
3273 }
3274
3275 static int amdgpu_device_ip_reinit_early_sriov(struct amdgpu_device *adev)
3276 {
3277         int i, r;
3278
3279         static enum amd_ip_block_type ip_order[] = {
3280                 AMD_IP_BLOCK_TYPE_COMMON,
3281                 AMD_IP_BLOCK_TYPE_GMC,
3282                 AMD_IP_BLOCK_TYPE_PSP,
3283                 AMD_IP_BLOCK_TYPE_IH,
3284         };
3285
3286         for (i = 0; i < adev->num_ip_blocks; i++) {
3287                 int j;
3288                 struct amdgpu_ip_block *block;
3289
3290                 block = &adev->ip_blocks[i];
3291                 block->status.hw = false;
3292
3293                 for (j = 0; j < ARRAY_SIZE(ip_order); j++) {
3294
3295                         if (block->version->type != ip_order[j] ||
3296                                 !block->status.valid)
3297                                 continue;
3298
3299                         r = block->version->funcs->hw_init(adev);
3300                         DRM_INFO("RE-INIT-early: %s %s\n", block->version->funcs->name, r?"failed":"succeeded");
3301                         if (r)
3302                                 return r;
3303                         block->status.hw = true;
3304                 }
3305         }
3306
3307         return 0;
3308 }
3309
3310 static int amdgpu_device_ip_reinit_late_sriov(struct amdgpu_device *adev)
3311 {
3312         int i, r;
3313
3314         static enum amd_ip_block_type ip_order[] = {
3315                 AMD_IP_BLOCK_TYPE_SMC,
3316                 AMD_IP_BLOCK_TYPE_DCE,
3317                 AMD_IP_BLOCK_TYPE_GFX,
3318                 AMD_IP_BLOCK_TYPE_SDMA,
3319                 AMD_IP_BLOCK_TYPE_MES,
3320                 AMD_IP_BLOCK_TYPE_UVD,
3321                 AMD_IP_BLOCK_TYPE_VCE,
3322                 AMD_IP_BLOCK_TYPE_VCN,
3323                 AMD_IP_BLOCK_TYPE_JPEG
3324         };
3325
3326         for (i = 0; i < ARRAY_SIZE(ip_order); i++) {
3327                 int j;
3328                 struct amdgpu_ip_block *block;
3329
3330                 for (j = 0; j < adev->num_ip_blocks; j++) {
3331                         block = &adev->ip_blocks[j];
3332
3333                         if (block->version->type != ip_order[i] ||
3334                                 !block->status.valid ||
3335                                 block->status.hw)
3336                                 continue;
3337
3338                         if (block->version->type == AMD_IP_BLOCK_TYPE_SMC)
3339                                 r = block->version->funcs->resume(adev);
3340                         else
3341                                 r = block->version->funcs->hw_init(adev);
3342
3343                         DRM_INFO("RE-INIT-late: %s %s\n", block->version->funcs->name, r?"failed":"succeeded");
3344                         if (r)
3345                                 return r;
3346                         block->status.hw = true;
3347                 }
3348         }
3349
3350         return 0;
3351 }
3352
3353 /**
3354  * amdgpu_device_ip_resume_phase1 - run resume for hardware IPs
3355  *
3356  * @adev: amdgpu_device pointer
3357  *
3358  * First resume function for hardware IPs.  The list of all the hardware
3359  * IPs that make up the asic is walked and the resume callbacks are run for
3360  * COMMON, GMC, and IH.  resume puts the hardware into a functional state
3361  * after a suspend and updates the software state as necessary.  This
3362  * function is also used for restoring the GPU after a GPU reset.
3363  * Returns 0 on success, negative error code on failure.
3364  */
3365 static int amdgpu_device_ip_resume_phase1(struct amdgpu_device *adev)
3366 {
3367         int i, r;
3368
3369         for (i = 0; i < adev->num_ip_blocks; i++) {
3370                 if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw)
3371                         continue;
3372                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3373                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3374                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||
3375                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP && amdgpu_sriov_vf(adev))) {
3376
3377                         r = adev->ip_blocks[i].version->funcs->resume(adev);
3378                         if (r) {
3379                                 DRM_ERROR("resume of IP block <%s> failed %d\n",
3380                                           adev->ip_blocks[i].version->funcs->name, r);
3381                                 return r;
3382                         }
3383                         adev->ip_blocks[i].status.hw = true;
3384                 }
3385         }
3386
3387         return 0;
3388 }
3389
3390 /**
3391  * amdgpu_device_ip_resume_phase2 - run resume for hardware IPs
3392  *
3393  * @adev: amdgpu_device pointer
3394  *
3395  * First resume function for hardware IPs.  The list of all the hardware
3396  * IPs that make up the asic is walked and the resume callbacks are run for
3397  * all blocks except COMMON, GMC, and IH.  resume puts the hardware into a
3398  * functional state after a suspend and updates the software state as
3399  * necessary.  This function is also used for restoring the GPU after a GPU
3400  * reset.
3401  * Returns 0 on success, negative error code on failure.
3402  */
3403 static int amdgpu_device_ip_resume_phase2(struct amdgpu_device *adev)
3404 {
3405         int i, r;
3406
3407         for (i = 0; i < adev->num_ip_blocks; i++) {
3408                 if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw)
3409                         continue;
3410                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3411                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3412                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||
3413                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)
3414                         continue;
3415                 r = adev->ip_blocks[i].version->funcs->resume(adev);
3416                 if (r) {
3417                         DRM_ERROR("resume of IP block <%s> failed %d\n",
3418                                   adev->ip_blocks[i].version->funcs->name, r);
3419                         return r;
3420                 }
3421                 adev->ip_blocks[i].status.hw = true;
3422         }
3423
3424         return 0;
3425 }
3426
3427 /**
3428  * amdgpu_device_ip_resume - run resume for hardware IPs
3429  *
3430  * @adev: amdgpu_device pointer
3431  *
3432  * Main resume function for hardware IPs.  The hardware IPs
3433  * are split into two resume functions because they are
3434  * also used in recovering from a GPU reset and some additional
3435  * steps need to be take between them.  In this case (S3/S4) they are
3436  * run sequentially.
3437  * Returns 0 on success, negative error code on failure.
3438  */
3439 static int amdgpu_device_ip_resume(struct amdgpu_device *adev)
3440 {
3441         int r;
3442
3443         r = amdgpu_device_ip_resume_phase1(adev);
3444         if (r)
3445                 return r;
3446
3447         r = amdgpu_device_fw_loading(adev);
3448         if (r)
3449                 return r;
3450
3451         r = amdgpu_device_ip_resume_phase2(adev);
3452
3453         if (adev->mman.buffer_funcs_ring->sched.ready)
3454                 amdgpu_ttm_set_buffer_funcs_status(adev, true);
3455
3456         return r;
3457 }
3458
3459 /**
3460  * amdgpu_device_detect_sriov_bios - determine if the board supports SR-IOV
3461  *
3462  * @adev: amdgpu_device pointer
3463  *
3464  * Query the VBIOS data tables to determine if the board supports SR-IOV.
3465  */
3466 static void amdgpu_device_detect_sriov_bios(struct amdgpu_device *adev)
3467 {
3468         if (amdgpu_sriov_vf(adev)) {
3469                 if (adev->is_atom_fw) {
3470                         if (amdgpu_atomfirmware_gpu_virtualization_supported(adev))
3471                                 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
3472                 } else {
3473                         if (amdgpu_atombios_has_gpu_virtualization_table(adev))
3474                                 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
3475                 }
3476
3477                 if (!(adev->virt.caps & AMDGPU_SRIOV_CAPS_SRIOV_VBIOS))
3478                         amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_NO_VBIOS, 0, 0);
3479         }
3480 }
3481
3482 /**
3483  * amdgpu_device_asic_has_dc_support - determine if DC supports the asic
3484  *
3485  * @asic_type: AMD asic type
3486  *
3487  * Check if there is DC (new modesetting infrastructre) support for an asic.
3488  * returns true if DC has support, false if not.
3489  */
3490 bool amdgpu_device_asic_has_dc_support(enum amd_asic_type asic_type)
3491 {
3492         switch (asic_type) {
3493 #ifdef CONFIG_DRM_AMDGPU_SI
3494         case CHIP_HAINAN:
3495 #endif
3496         case CHIP_TOPAZ:
3497                 /* chips with no display hardware */
3498                 return false;
3499 #if defined(CONFIG_DRM_AMD_DC)
3500         case CHIP_TAHITI:
3501         case CHIP_PITCAIRN:
3502         case CHIP_VERDE:
3503         case CHIP_OLAND:
3504                 /*
3505                  * We have systems in the wild with these ASICs that require
3506                  * LVDS and VGA support which is not supported with DC.
3507                  *
3508                  * Fallback to the non-DC driver here by default so as not to
3509                  * cause regressions.
3510                  */
3511 #if defined(CONFIG_DRM_AMD_DC_SI)
3512                 return amdgpu_dc > 0;
3513 #else
3514                 return false;
3515 #endif
3516         case CHIP_BONAIRE:
3517         case CHIP_KAVERI:
3518         case CHIP_KABINI:
3519         case CHIP_MULLINS:
3520                 /*
3521                  * We have systems in the wild with these ASICs that require
3522                  * VGA support which is not supported with DC.
3523                  *
3524                  * Fallback to the non-DC driver here by default so as not to
3525                  * cause regressions.
3526                  */
3527                 return amdgpu_dc > 0;
3528         default:
3529                 return amdgpu_dc != 0;
3530 #else
3531         default:
3532                 if (amdgpu_dc > 0)
3533                         DRM_INFO_ONCE("Display Core has been requested via kernel parameter but isn't supported by ASIC, ignoring\n");
3534                 return false;
3535 #endif
3536         }
3537 }
3538
3539 /**
3540  * amdgpu_device_has_dc_support - check if dc is supported
3541  *
3542  * @adev: amdgpu_device pointer
3543  *
3544  * Returns true for supported, false for not supported
3545  */
3546 bool amdgpu_device_has_dc_support(struct amdgpu_device *adev)
3547 {
3548         if (adev->enable_virtual_display ||
3549             (adev->harvest_ip_mask & AMD_HARVEST_IP_DMU_MASK))
3550                 return false;
3551
3552         return amdgpu_device_asic_has_dc_support(adev->asic_type);
3553 }
3554
3555 static void amdgpu_device_xgmi_reset_func(struct work_struct *__work)
3556 {
3557         struct amdgpu_device *adev =
3558                 container_of(__work, struct amdgpu_device, xgmi_reset_work);
3559         struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
3560
3561         /* It's a bug to not have a hive within this function */
3562         if (WARN_ON(!hive))
3563                 return;
3564
3565         /*
3566          * Use task barrier to synchronize all xgmi reset works across the
3567          * hive. task_barrier_enter and task_barrier_exit will block
3568          * until all the threads running the xgmi reset works reach
3569          * those points. task_barrier_full will do both blocks.
3570          */
3571         if (amdgpu_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) {
3572
3573                 task_barrier_enter(&hive->tb);
3574                 adev->asic_reset_res = amdgpu_device_baco_enter(adev_to_drm(adev));
3575
3576                 if (adev->asic_reset_res)
3577                         goto fail;
3578
3579                 task_barrier_exit(&hive->tb);
3580                 adev->asic_reset_res = amdgpu_device_baco_exit(adev_to_drm(adev));
3581
3582                 if (adev->asic_reset_res)
3583                         goto fail;
3584
3585                 amdgpu_ras_reset_error_count(adev, AMDGPU_RAS_BLOCK__MMHUB);
3586         } else {
3587
3588                 task_barrier_full(&hive->tb);
3589                 adev->asic_reset_res =  amdgpu_asic_reset(adev);
3590         }
3591
3592 fail:
3593         if (adev->asic_reset_res)
3594                 DRM_WARN("ASIC reset failed with error, %d for drm dev, %s",
3595                          adev->asic_reset_res, adev_to_drm(adev)->unique);
3596         amdgpu_put_xgmi_hive(hive);
3597 }
3598
3599 static int amdgpu_device_get_job_timeout_settings(struct amdgpu_device *adev)
3600 {
3601         char *input = amdgpu_lockup_timeout;
3602         char *timeout_setting = NULL;
3603         int index = 0;
3604         long timeout;
3605         int ret = 0;
3606
3607         /*
3608          * By default timeout for non compute jobs is 10000
3609          * and 60000 for compute jobs.
3610          * In SR-IOV or passthrough mode, timeout for compute
3611          * jobs are 60000 by default.
3612          */
3613         adev->gfx_timeout = msecs_to_jiffies(10000);
3614         adev->sdma_timeout = adev->video_timeout = adev->gfx_timeout;
3615         if (amdgpu_sriov_vf(adev))
3616                 adev->compute_timeout = amdgpu_sriov_is_pp_one_vf(adev) ?
3617                                         msecs_to_jiffies(60000) : msecs_to_jiffies(10000);
3618         else
3619                 adev->compute_timeout =  msecs_to_jiffies(60000);
3620
3621         if (strnlen(input, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH)) {
3622                 while ((timeout_setting = strsep(&input, ",")) &&
3623                                 strnlen(timeout_setting, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH)) {
3624                         ret = kstrtol(timeout_setting, 0, &timeout);
3625                         if (ret)
3626                                 return ret;
3627
3628                         if (timeout == 0) {
3629                                 index++;
3630                                 continue;
3631                         } else if (timeout < 0) {
3632                                 timeout = MAX_SCHEDULE_TIMEOUT;
3633                                 dev_warn(adev->dev, "lockup timeout disabled");
3634                                 add_taint(TAINT_SOFTLOCKUP, LOCKDEP_STILL_OK);
3635                         } else {
3636                                 timeout = msecs_to_jiffies(timeout);
3637                         }
3638
3639                         switch (index++) {
3640                         case 0:
3641                                 adev->gfx_timeout = timeout;
3642                                 break;
3643                         case 1:
3644                                 adev->compute_timeout = timeout;
3645                                 break;
3646                         case 2:
3647                                 adev->sdma_timeout = timeout;
3648                                 break;
3649                         case 3:
3650                                 adev->video_timeout = timeout;
3651                                 break;
3652                         default:
3653                                 break;
3654                         }
3655                 }
3656                 /*
3657                  * There is only one value specified and
3658                  * it should apply to all non-compute jobs.
3659                  */
3660                 if (index == 1) {
3661                         adev->sdma_timeout = adev->video_timeout = adev->gfx_timeout;
3662                         if (amdgpu_sriov_vf(adev) || amdgpu_passthrough(adev))
3663                                 adev->compute_timeout = adev->gfx_timeout;
3664                 }
3665         }
3666
3667         return ret;
3668 }
3669
3670 /**
3671  * amdgpu_device_check_iommu_direct_map - check if RAM direct mapped to GPU
3672  *
3673  * @adev: amdgpu_device pointer
3674  *
3675  * RAM direct mapped to GPU if IOMMU is not enabled or is pass through mode
3676  */
3677 static void amdgpu_device_check_iommu_direct_map(struct amdgpu_device *adev)
3678 {
3679         struct iommu_domain *domain;
3680
3681         domain = iommu_get_domain_for_dev(adev->dev);
3682         if (!domain || domain->type == IOMMU_DOMAIN_IDENTITY)
3683                 adev->ram_is_direct_mapped = true;
3684 }
3685
3686 static const struct attribute *amdgpu_dev_attributes[] = {
3687         &dev_attr_pcie_replay_count.attr,
3688         NULL
3689 };
3690
3691 static void amdgpu_device_set_mcbp(struct amdgpu_device *adev)
3692 {
3693         if (amdgpu_mcbp == 1)
3694                 adev->gfx.mcbp = true;
3695         else if (amdgpu_mcbp == 0)
3696                 adev->gfx.mcbp = false;
3697         else if ((amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(9, 0, 0)) &&
3698                  (amdgpu_ip_version(adev, GC_HWIP, 0) < IP_VERSION(10, 0, 0)) &&
3699                  adev->gfx.num_gfx_rings)
3700                 adev->gfx.mcbp = true;
3701
3702         if (amdgpu_sriov_vf(adev))
3703                 adev->gfx.mcbp = true;
3704
3705         if (adev->gfx.mcbp)
3706                 DRM_INFO("MCBP is enabled\n");
3707 }
3708
3709 /**
3710  * amdgpu_device_init - initialize the driver
3711  *
3712  * @adev: amdgpu_device pointer
3713  * @flags: driver flags
3714  *
3715  * Initializes the driver info and hw (all asics).
3716  * Returns 0 for success or an error on failure.
3717  * Called at driver startup.
3718  */
3719 int amdgpu_device_init(struct amdgpu_device *adev,
3720                        uint32_t flags)
3721 {
3722         struct drm_device *ddev = adev_to_drm(adev);
3723         struct pci_dev *pdev = adev->pdev;
3724         int r, i;
3725         bool px = false;
3726         u32 max_MBps;
3727         int tmp;
3728
3729         adev->shutdown = false;
3730         adev->flags = flags;
3731
3732         if (amdgpu_force_asic_type >= 0 && amdgpu_force_asic_type < CHIP_LAST)
3733                 adev->asic_type = amdgpu_force_asic_type;
3734         else
3735                 adev->asic_type = flags & AMD_ASIC_MASK;
3736
3737         adev->usec_timeout = AMDGPU_MAX_USEC_TIMEOUT;
3738         if (amdgpu_emu_mode == 1)
3739                 adev->usec_timeout *= 10;
3740         adev->gmc.gart_size = 512 * 1024 * 1024;
3741         adev->accel_working = false;
3742         adev->num_rings = 0;
3743         RCU_INIT_POINTER(adev->gang_submit, dma_fence_get_stub());
3744         adev->mman.buffer_funcs = NULL;
3745         adev->mman.buffer_funcs_ring = NULL;
3746         adev->vm_manager.vm_pte_funcs = NULL;
3747         adev->vm_manager.vm_pte_num_scheds = 0;
3748         adev->gmc.gmc_funcs = NULL;
3749         adev->harvest_ip_mask = 0x0;
3750         adev->fence_context = dma_fence_context_alloc(AMDGPU_MAX_RINGS);
3751         bitmap_zero(adev->gfx.pipe_reserve_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
3752
3753         adev->smc_rreg = &amdgpu_invalid_rreg;
3754         adev->smc_wreg = &amdgpu_invalid_wreg;
3755         adev->pcie_rreg = &amdgpu_invalid_rreg;
3756         adev->pcie_wreg = &amdgpu_invalid_wreg;
3757         adev->pcie_rreg_ext = &amdgpu_invalid_rreg_ext;
3758         adev->pcie_wreg_ext = &amdgpu_invalid_wreg_ext;
3759         adev->pciep_rreg = &amdgpu_invalid_rreg;
3760         adev->pciep_wreg = &amdgpu_invalid_wreg;
3761         adev->pcie_rreg64 = &amdgpu_invalid_rreg64;
3762         adev->pcie_wreg64 = &amdgpu_invalid_wreg64;
3763         adev->pcie_rreg64_ext = &amdgpu_invalid_rreg64_ext;
3764         adev->pcie_wreg64_ext = &amdgpu_invalid_wreg64_ext;
3765         adev->uvd_ctx_rreg = &amdgpu_invalid_rreg;
3766         adev->uvd_ctx_wreg = &amdgpu_invalid_wreg;
3767         adev->didt_rreg = &amdgpu_invalid_rreg;
3768         adev->didt_wreg = &amdgpu_invalid_wreg;
3769         adev->gc_cac_rreg = &amdgpu_invalid_rreg;
3770         adev->gc_cac_wreg = &amdgpu_invalid_wreg;
3771         adev->audio_endpt_rreg = &amdgpu_block_invalid_rreg;
3772         adev->audio_endpt_wreg = &amdgpu_block_invalid_wreg;
3773
3774         DRM_INFO("initializing kernel modesetting (%s 0x%04X:0x%04X 0x%04X:0x%04X 0x%02X).\n",
3775                  amdgpu_asic_name[adev->asic_type], pdev->vendor, pdev->device,
3776                  pdev->subsystem_vendor, pdev->subsystem_device, pdev->revision);
3777
3778         /* mutex initialization are all done here so we
3779          * can recall function without having locking issues
3780          */
3781         mutex_init(&adev->firmware.mutex);
3782         mutex_init(&adev->pm.mutex);
3783         mutex_init(&adev->gfx.gpu_clock_mutex);
3784         mutex_init(&adev->srbm_mutex);
3785         mutex_init(&adev->gfx.pipe_reserve_mutex);
3786         mutex_init(&adev->gfx.gfx_off_mutex);
3787         mutex_init(&adev->gfx.partition_mutex);
3788         mutex_init(&adev->grbm_idx_mutex);
3789         mutex_init(&adev->mn_lock);
3790         mutex_init(&adev->virt.vf_errors.lock);
3791         hash_init(adev->mn_hash);
3792         mutex_init(&adev->psp.mutex);
3793         mutex_init(&adev->notifier_lock);
3794         mutex_init(&adev->pm.stable_pstate_ctx_lock);
3795         mutex_init(&adev->benchmark_mutex);
3796
3797         amdgpu_device_init_apu_flags(adev);
3798
3799         r = amdgpu_device_check_arguments(adev);
3800         if (r)
3801                 return r;
3802
3803         spin_lock_init(&adev->mmio_idx_lock);
3804         spin_lock_init(&adev->smc_idx_lock);
3805         spin_lock_init(&adev->pcie_idx_lock);
3806         spin_lock_init(&adev->uvd_ctx_idx_lock);
3807         spin_lock_init(&adev->didt_idx_lock);
3808         spin_lock_init(&adev->gc_cac_idx_lock);
3809         spin_lock_init(&adev->se_cac_idx_lock);
3810         spin_lock_init(&adev->audio_endpt_idx_lock);
3811         spin_lock_init(&adev->mm_stats.lock);
3812
3813         INIT_LIST_HEAD(&adev->shadow_list);
3814         mutex_init(&adev->shadow_list_lock);
3815
3816         INIT_LIST_HEAD(&adev->reset_list);
3817
3818         INIT_LIST_HEAD(&adev->ras_list);
3819
3820         INIT_LIST_HEAD(&adev->pm.od_kobj_list);
3821
3822         INIT_DELAYED_WORK(&adev->delayed_init_work,
3823                           amdgpu_device_delayed_init_work_handler);
3824         INIT_DELAYED_WORK(&adev->gfx.gfx_off_delay_work,
3825                           amdgpu_device_delay_enable_gfx_off);
3826
3827         INIT_WORK(&adev->xgmi_reset_work, amdgpu_device_xgmi_reset_func);
3828
3829         adev->gfx.gfx_off_req_count = 1;
3830         adev->gfx.gfx_off_residency = 0;
3831         adev->gfx.gfx_off_entrycount = 0;
3832         adev->pm.ac_power = power_supply_is_system_supplied() > 0;
3833
3834         atomic_set(&adev->throttling_logging_enabled, 1);
3835         /*
3836          * If throttling continues, logging will be performed every minute
3837          * to avoid log flooding. "-1" is subtracted since the thermal
3838          * throttling interrupt comes every second. Thus, the total logging
3839          * interval is 59 seconds(retelimited printk interval) + 1(waiting
3840          * for throttling interrupt) = 60 seconds.
3841          */
3842         ratelimit_state_init(&adev->throttling_logging_rs, (60 - 1) * HZ, 1);
3843         ratelimit_set_flags(&adev->throttling_logging_rs, RATELIMIT_MSG_ON_RELEASE);
3844
3845         /* Registers mapping */
3846         /* TODO: block userspace mapping of io register */
3847         if (adev->asic_type >= CHIP_BONAIRE) {
3848                 adev->rmmio_base = pci_resource_start(adev->pdev, 5);
3849                 adev->rmmio_size = pci_resource_len(adev->pdev, 5);
3850         } else {
3851                 adev->rmmio_base = pci_resource_start(adev->pdev, 2);
3852                 adev->rmmio_size = pci_resource_len(adev->pdev, 2);
3853         }
3854
3855         for (i = 0; i < AMD_IP_BLOCK_TYPE_NUM; i++)
3856                 atomic_set(&adev->pm.pwr_state[i], POWER_STATE_UNKNOWN);
3857
3858         adev->rmmio = ioremap(adev->rmmio_base, adev->rmmio_size);
3859         if (!adev->rmmio)
3860                 return -ENOMEM;
3861
3862         DRM_INFO("register mmio base: 0x%08X\n", (uint32_t)adev->rmmio_base);
3863         DRM_INFO("register mmio size: %u\n", (unsigned int)adev->rmmio_size);
3864
3865         /*
3866          * Reset domain needs to be present early, before XGMI hive discovered
3867          * (if any) and intitialized to use reset sem and in_gpu reset flag
3868          * early on during init and before calling to RREG32.
3869          */
3870         adev->reset_domain = amdgpu_reset_create_reset_domain(SINGLE_DEVICE, "amdgpu-reset-dev");
3871         if (!adev->reset_domain)
3872                 return -ENOMEM;
3873
3874         /* detect hw virtualization here */
3875         amdgpu_detect_virtualization(adev);
3876
3877         amdgpu_device_get_pcie_info(adev);
3878
3879         r = amdgpu_device_get_job_timeout_settings(adev);
3880         if (r) {
3881                 dev_err(adev->dev, "invalid lockup_timeout parameter syntax\n");
3882                 return r;
3883         }
3884
3885         /* early init functions */
3886         r = amdgpu_device_ip_early_init(adev);
3887         if (r)
3888                 return r;
3889
3890         amdgpu_device_set_mcbp(adev);
3891
3892         /* Get rid of things like offb */
3893         r = drm_aperture_remove_conflicting_pci_framebuffers(adev->pdev, &amdgpu_kms_driver);
3894         if (r)
3895                 return r;
3896
3897         /* Enable TMZ based on IP_VERSION */
3898         amdgpu_gmc_tmz_set(adev);
3899
3900         amdgpu_gmc_noretry_set(adev);
3901         /* Need to get xgmi info early to decide the reset behavior*/
3902         if (adev->gmc.xgmi.supported) {
3903                 r = adev->gfxhub.funcs->get_xgmi_info(adev);
3904                 if (r)
3905                         return r;
3906         }
3907
3908         /* enable PCIE atomic ops */
3909         if (amdgpu_sriov_vf(adev)) {
3910                 if (adev->virt.fw_reserve.p_pf2vf)
3911                         adev->have_atomics_support = ((struct amd_sriov_msg_pf2vf_info *)
3912                                                       adev->virt.fw_reserve.p_pf2vf)->pcie_atomic_ops_support_flags ==
3913                                 (PCI_EXP_DEVCAP2_ATOMIC_COMP32 | PCI_EXP_DEVCAP2_ATOMIC_COMP64);
3914         /* APUs w/ gfx9 onwards doesn't reply on PCIe atomics, rather it is a
3915          * internal path natively support atomics, set have_atomics_support to true.
3916          */
3917         } else if ((adev->flags & AMD_IS_APU) &&
3918                    (amdgpu_ip_version(adev, GC_HWIP, 0) >
3919                     IP_VERSION(9, 0, 0))) {
3920                 adev->have_atomics_support = true;
3921         } else {
3922                 adev->have_atomics_support =
3923                         !pci_enable_atomic_ops_to_root(adev->pdev,
3924                                           PCI_EXP_DEVCAP2_ATOMIC_COMP32 |
3925                                           PCI_EXP_DEVCAP2_ATOMIC_COMP64);
3926         }
3927
3928         if (!adev->have_atomics_support)
3929                 dev_info(adev->dev, "PCIE atomic ops is not supported\n");
3930
3931         /* doorbell bar mapping and doorbell index init*/
3932         amdgpu_doorbell_init(adev);
3933
3934         if (amdgpu_emu_mode == 1) {
3935                 /* post the asic on emulation mode */
3936                 emu_soc_asic_init(adev);
3937                 goto fence_driver_init;
3938         }
3939
3940         amdgpu_reset_init(adev);
3941
3942         /* detect if we are with an SRIOV vbios */
3943         if (adev->bios)
3944                 amdgpu_device_detect_sriov_bios(adev);
3945
3946         /* check if we need to reset the asic
3947          *  E.g., driver was not cleanly unloaded previously, etc.
3948          */
3949         if (!amdgpu_sriov_vf(adev) && amdgpu_asic_need_reset_on_init(adev)) {
3950                 if (adev->gmc.xgmi.num_physical_nodes) {
3951                         dev_info(adev->dev, "Pending hive reset.\n");
3952                         adev->gmc.xgmi.pending_reset = true;
3953                         /* Only need to init necessary block for SMU to handle the reset */
3954                         for (i = 0; i < adev->num_ip_blocks; i++) {
3955                                 if (!adev->ip_blocks[i].status.valid)
3956                                         continue;
3957                                 if (!(adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3958                                       adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3959                                       adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||
3960                                       adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC)) {
3961                                         DRM_DEBUG("IP %s disabled for hw_init.\n",
3962                                                 adev->ip_blocks[i].version->funcs->name);
3963                                         adev->ip_blocks[i].status.hw = true;
3964                                 }
3965                         }
3966                 } else {
3967                         switch (amdgpu_ip_version(adev, MP1_HWIP, 0)) {
3968                         case IP_VERSION(13, 0, 0):
3969                         case IP_VERSION(13, 0, 7):
3970                         case IP_VERSION(13, 0, 10):
3971                                 r = psp_gpu_reset(adev);
3972                                 break;
3973                         default:
3974                                 tmp = amdgpu_reset_method;
3975                                 /* It should do a default reset when loading or reloading the driver,
3976                                  * regardless of the module parameter reset_method.
3977                                  */
3978                                 amdgpu_reset_method = AMD_RESET_METHOD_NONE;
3979                                 r = amdgpu_asic_reset(adev);
3980                                 amdgpu_reset_method = tmp;
3981                                 break;
3982                         }
3983
3984                         if (r) {
3985                                 dev_err(adev->dev, "asic reset on init failed\n");
3986                                 goto failed;
3987                         }
3988                 }
3989         }
3990
3991         /* Post card if necessary */
3992         if (amdgpu_device_need_post(adev)) {
3993                 if (!adev->bios) {
3994                         dev_err(adev->dev, "no vBIOS found\n");
3995                         r = -EINVAL;
3996                         goto failed;
3997                 }
3998                 DRM_INFO("GPU posting now...\n");
3999                 r = amdgpu_device_asic_init(adev);
4000                 if (r) {
4001                         dev_err(adev->dev, "gpu post error!\n");
4002                         goto failed;
4003                 }
4004         }
4005
4006         if (adev->bios) {
4007                 if (adev->is_atom_fw) {
4008                         /* Initialize clocks */
4009                         r = amdgpu_atomfirmware_get_clock_info(adev);
4010                         if (r) {
4011                                 dev_err(adev->dev, "amdgpu_atomfirmware_get_clock_info failed\n");
4012                                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
4013                                 goto failed;
4014                         }
4015                 } else {
4016                         /* Initialize clocks */
4017                         r = amdgpu_atombios_get_clock_info(adev);
4018                         if (r) {
4019                                 dev_err(adev->dev, "amdgpu_atombios_get_clock_info failed\n");
4020                                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
4021                                 goto failed;
4022                         }
4023                         /* init i2c buses */
4024                         if (!amdgpu_device_has_dc_support(adev))
4025                                 amdgpu_atombios_i2c_init(adev);
4026                 }
4027         }
4028
4029 fence_driver_init:
4030         /* Fence driver */
4031         r = amdgpu_fence_driver_sw_init(adev);
4032         if (r) {
4033                 dev_err(adev->dev, "amdgpu_fence_driver_sw_init failed\n");
4034                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_FENCE_INIT_FAIL, 0, 0);
4035                 goto failed;
4036         }
4037
4038         /* init the mode config */
4039         drm_mode_config_init(adev_to_drm(adev));
4040
4041         r = amdgpu_device_ip_init(adev);
4042         if (r) {
4043                 dev_err(adev->dev, "amdgpu_device_ip_init failed\n");
4044                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_INIT_FAIL, 0, 0);
4045                 goto release_ras_con;
4046         }
4047
4048         amdgpu_fence_driver_hw_init(adev);
4049
4050         dev_info(adev->dev,
4051                 "SE %d, SH per SE %d, CU per SH %d, active_cu_number %d\n",
4052                         adev->gfx.config.max_shader_engines,
4053                         adev->gfx.config.max_sh_per_se,
4054                         adev->gfx.config.max_cu_per_sh,
4055                         adev->gfx.cu_info.number);
4056
4057         adev->accel_working = true;
4058
4059         amdgpu_vm_check_compute_bug(adev);
4060
4061         /* Initialize the buffer migration limit. */
4062         if (amdgpu_moverate >= 0)
4063                 max_MBps = amdgpu_moverate;
4064         else
4065                 max_MBps = 8; /* Allow 8 MB/s. */
4066         /* Get a log2 for easy divisions. */
4067         adev->mm_stats.log2_max_MBps = ilog2(max(1u, max_MBps));
4068
4069         /*
4070          * Register gpu instance before amdgpu_device_enable_mgpu_fan_boost.
4071          * Otherwise the mgpu fan boost feature will be skipped due to the
4072          * gpu instance is counted less.
4073          */
4074         amdgpu_register_gpu_instance(adev);
4075
4076         /* enable clockgating, etc. after ib tests, etc. since some blocks require
4077          * explicit gating rather than handling it automatically.
4078          */
4079         if (!adev->gmc.xgmi.pending_reset) {
4080                 r = amdgpu_device_ip_late_init(adev);
4081                 if (r) {
4082                         dev_err(adev->dev, "amdgpu_device_ip_late_init failed\n");
4083                         amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_LATE_INIT_FAIL, 0, r);
4084                         goto release_ras_con;
4085                 }
4086                 /* must succeed. */
4087                 amdgpu_ras_resume(adev);
4088                 queue_delayed_work(system_wq, &adev->delayed_init_work,
4089                                    msecs_to_jiffies(AMDGPU_RESUME_MS));
4090         }
4091
4092         if (amdgpu_sriov_vf(adev)) {
4093                 amdgpu_virt_release_full_gpu(adev, true);
4094                 flush_delayed_work(&adev->delayed_init_work);
4095         }
4096
4097         /*
4098          * Place those sysfs registering after `late_init`. As some of those
4099          * operations performed in `late_init` might affect the sysfs
4100          * interfaces creating.
4101          */
4102         r = amdgpu_atombios_sysfs_init(adev);
4103         if (r)
4104                 drm_err(&adev->ddev,
4105                         "registering atombios sysfs failed (%d).\n", r);
4106
4107         r = amdgpu_pm_sysfs_init(adev);
4108         if (r)
4109                 DRM_ERROR("registering pm sysfs failed (%d).\n", r);
4110
4111         r = amdgpu_ucode_sysfs_init(adev);
4112         if (r) {
4113                 adev->ucode_sysfs_en = false;
4114                 DRM_ERROR("Creating firmware sysfs failed (%d).\n", r);
4115         } else
4116                 adev->ucode_sysfs_en = true;
4117
4118         r = sysfs_create_files(&adev->dev->kobj, amdgpu_dev_attributes);
4119         if (r)
4120                 dev_err(adev->dev, "Could not create amdgpu device attr\n");
4121
4122         r = devm_device_add_group(adev->dev, &amdgpu_board_attrs_group);
4123         if (r)
4124                 dev_err(adev->dev,
4125                         "Could not create amdgpu board attributes\n");
4126
4127         amdgpu_fru_sysfs_init(adev);
4128
4129         if (IS_ENABLED(CONFIG_PERF_EVENTS))
4130                 r = amdgpu_pmu_init(adev);
4131         if (r)
4132                 dev_err(adev->dev, "amdgpu_pmu_init failed\n");
4133
4134         /* Have stored pci confspace at hand for restore in sudden PCI error */
4135         if (amdgpu_device_cache_pci_state(adev->pdev))
4136                 pci_restore_state(pdev);
4137
4138         /* if we have > 1 VGA cards, then disable the amdgpu VGA resources */
4139         /* this will fail for cards that aren't VGA class devices, just
4140          * ignore it
4141          */
4142         if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA)
4143                 vga_client_register(adev->pdev, amdgpu_device_vga_set_decode);
4144
4145         px = amdgpu_device_supports_px(ddev);
4146
4147         if (px || (!dev_is_removable(&adev->pdev->dev) &&
4148                                 apple_gmux_detect(NULL, NULL)))
4149                 vga_switcheroo_register_client(adev->pdev,
4150                                                &amdgpu_switcheroo_ops, px);
4151
4152         if (px)
4153                 vga_switcheroo_init_domain_pm_ops(adev->dev, &adev->vga_pm_domain);
4154
4155         if (adev->gmc.xgmi.pending_reset)
4156                 queue_delayed_work(system_wq, &mgpu_info.delayed_reset_work,
4157                                    msecs_to_jiffies(AMDGPU_RESUME_MS));
4158
4159         amdgpu_device_check_iommu_direct_map(adev);
4160
4161         return 0;
4162
4163 release_ras_con:
4164         if (amdgpu_sriov_vf(adev))
4165                 amdgpu_virt_release_full_gpu(adev, true);
4166
4167         /* failed in exclusive mode due to timeout */
4168         if (amdgpu_sriov_vf(adev) &&
4169                 !amdgpu_sriov_runtime(adev) &&
4170                 amdgpu_virt_mmio_blocked(adev) &&
4171                 !amdgpu_virt_wait_reset(adev)) {
4172                 dev_err(adev->dev, "VF exclusive mode timeout\n");
4173                 /* Don't send request since VF is inactive. */
4174                 adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
4175                 adev->virt.ops = NULL;
4176                 r = -EAGAIN;
4177         }
4178         amdgpu_release_ras_context(adev);
4179
4180 failed:
4181         amdgpu_vf_error_trans_all(adev);
4182
4183         return r;
4184 }
4185
4186 static void amdgpu_device_unmap_mmio(struct amdgpu_device *adev)
4187 {
4188
4189         /* Clear all CPU mappings pointing to this device */
4190         unmap_mapping_range(adev->ddev.anon_inode->i_mapping, 0, 0, 1);
4191
4192         /* Unmap all mapped bars - Doorbell, registers and VRAM */
4193         amdgpu_doorbell_fini(adev);
4194
4195         iounmap(adev->rmmio);
4196         adev->rmmio = NULL;
4197         if (adev->mman.aper_base_kaddr)
4198                 iounmap(adev->mman.aper_base_kaddr);
4199         adev->mman.aper_base_kaddr = NULL;
4200
4201         /* Memory manager related */
4202         if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu) {
4203                 arch_phys_wc_del(adev->gmc.vram_mtrr);
4204                 arch_io_free_memtype_wc(adev->gmc.aper_base, adev->gmc.aper_size);
4205         }
4206 }
4207
4208 /**
4209  * amdgpu_device_fini_hw - tear down the driver
4210  *
4211  * @adev: amdgpu_device pointer
4212  *
4213  * Tear down the driver info (all asics).
4214  * Called at driver shutdown.
4215  */
4216 void amdgpu_device_fini_hw(struct amdgpu_device *adev)
4217 {
4218         dev_info(adev->dev, "amdgpu: finishing device.\n");
4219         flush_delayed_work(&adev->delayed_init_work);
4220         adev->shutdown = true;
4221
4222         /* make sure IB test finished before entering exclusive mode
4223          * to avoid preemption on IB test
4224          */
4225         if (amdgpu_sriov_vf(adev)) {
4226                 amdgpu_virt_request_full_gpu(adev, false);
4227                 amdgpu_virt_fini_data_exchange(adev);
4228         }
4229
4230         /* disable all interrupts */
4231         amdgpu_irq_disable_all(adev);
4232         if (adev->mode_info.mode_config_initialized) {
4233                 if (!drm_drv_uses_atomic_modeset(adev_to_drm(adev)))
4234                         drm_helper_force_disable_all(adev_to_drm(adev));
4235                 else
4236                         drm_atomic_helper_shutdown(adev_to_drm(adev));
4237         }
4238         amdgpu_fence_driver_hw_fini(adev);
4239
4240         if (adev->mman.initialized)
4241                 drain_workqueue(adev->mman.bdev.wq);
4242
4243         if (adev->pm.sysfs_initialized)
4244                 amdgpu_pm_sysfs_fini(adev);
4245         if (adev->ucode_sysfs_en)
4246                 amdgpu_ucode_sysfs_fini(adev);
4247         sysfs_remove_files(&adev->dev->kobj, amdgpu_dev_attributes);
4248         amdgpu_fru_sysfs_fini(adev);
4249
4250         /* disable ras feature must before hw fini */
4251         amdgpu_ras_pre_fini(adev);
4252
4253         amdgpu_ttm_set_buffer_funcs_status(adev, false);
4254
4255         amdgpu_device_ip_fini_early(adev);
4256
4257         amdgpu_irq_fini_hw(adev);
4258
4259         if (adev->mman.initialized)
4260                 ttm_device_clear_dma_mappings(&adev->mman.bdev);
4261
4262         amdgpu_gart_dummy_page_fini(adev);
4263
4264         if (drm_dev_is_unplugged(adev_to_drm(adev)))
4265                 amdgpu_device_unmap_mmio(adev);
4266
4267 }
4268
4269 void amdgpu_device_fini_sw(struct amdgpu_device *adev)
4270 {
4271         int idx;
4272         bool px;
4273
4274         amdgpu_fence_driver_sw_fini(adev);
4275         amdgpu_device_ip_fini(adev);
4276         amdgpu_ucode_release(&adev->firmware.gpu_info_fw);
4277         adev->accel_working = false;
4278         dma_fence_put(rcu_dereference_protected(adev->gang_submit, true));
4279
4280         amdgpu_reset_fini(adev);
4281
4282         /* free i2c buses */
4283         if (!amdgpu_device_has_dc_support(adev))
4284                 amdgpu_i2c_fini(adev);
4285
4286         if (amdgpu_emu_mode != 1)
4287                 amdgpu_atombios_fini(adev);
4288
4289         kfree(adev->bios);
4290         adev->bios = NULL;
4291
4292         kfree(adev->fru_info);
4293         adev->fru_info = NULL;
4294
4295         px = amdgpu_device_supports_px(adev_to_drm(adev));
4296
4297         if (px || (!dev_is_removable(&adev->pdev->dev) &&
4298                                 apple_gmux_detect(NULL, NULL)))
4299                 vga_switcheroo_unregister_client(adev->pdev);
4300
4301         if (px)
4302                 vga_switcheroo_fini_domain_pm_ops(adev->dev);
4303
4304         if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA)
4305                 vga_client_unregister(adev->pdev);
4306
4307         if (drm_dev_enter(adev_to_drm(adev), &idx)) {
4308
4309                 iounmap(adev->rmmio);
4310                 adev->rmmio = NULL;
4311                 amdgpu_doorbell_fini(adev);
4312                 drm_dev_exit(idx);
4313         }
4314
4315         if (IS_ENABLED(CONFIG_PERF_EVENTS))
4316                 amdgpu_pmu_fini(adev);
4317         if (adev->mman.discovery_bin)
4318                 amdgpu_discovery_fini(adev);
4319
4320         amdgpu_reset_put_reset_domain(adev->reset_domain);
4321         adev->reset_domain = NULL;
4322
4323         kfree(adev->pci_state);
4324
4325 }
4326
4327 /**
4328  * amdgpu_device_evict_resources - evict device resources
4329  * @adev: amdgpu device object
4330  *
4331  * Evicts all ttm device resources(vram BOs, gart table) from the lru list
4332  * of the vram memory type. Mainly used for evicting device resources
4333  * at suspend time.
4334  *
4335  */
4336 static int amdgpu_device_evict_resources(struct amdgpu_device *adev)
4337 {
4338         int ret;
4339
4340         /* No need to evict vram on APUs for suspend to ram or s2idle */
4341         if ((adev->in_s3 || adev->in_s0ix) && (adev->flags & AMD_IS_APU))
4342                 return 0;
4343
4344         ret = amdgpu_ttm_evict_resources(adev, TTM_PL_VRAM);
4345         if (ret)
4346                 DRM_WARN("evicting device resources failed\n");
4347         return ret;
4348 }
4349
4350 /*
4351  * Suspend & resume.
4352  */
4353 /**
4354  * amdgpu_device_prepare - prepare for device suspend
4355  *
4356  * @dev: drm dev pointer
4357  *
4358  * Prepare to put the hw in the suspend state (all asics).
4359  * Returns 0 for success or an error on failure.
4360  * Called at driver suspend.
4361  */
4362 int amdgpu_device_prepare(struct drm_device *dev)
4363 {
4364         struct amdgpu_device *adev = drm_to_adev(dev);
4365         int i, r;
4366
4367         if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
4368                 return 0;
4369
4370         /* Evict the majority of BOs before starting suspend sequence */
4371         r = amdgpu_device_evict_resources(adev);
4372         if (r)
4373                 return r;
4374
4375         for (i = 0; i < adev->num_ip_blocks; i++) {
4376                 if (!adev->ip_blocks[i].status.valid)
4377                         continue;
4378                 if (!adev->ip_blocks[i].version->funcs->prepare_suspend)
4379                         continue;
4380                 r = adev->ip_blocks[i].version->funcs->prepare_suspend((void *)adev);
4381                 if (r)
4382                         return r;
4383         }
4384
4385         return 0;
4386 }
4387
4388 /**
4389  * amdgpu_device_suspend - initiate device suspend
4390  *
4391  * @dev: drm dev pointer
4392  * @fbcon : notify the fbdev of suspend
4393  *
4394  * Puts the hw in the suspend state (all asics).
4395  * Returns 0 for success or an error on failure.
4396  * Called at driver suspend.
4397  */
4398 int amdgpu_device_suspend(struct drm_device *dev, bool fbcon)
4399 {
4400         struct amdgpu_device *adev = drm_to_adev(dev);
4401         int r = 0;
4402
4403         if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
4404                 return 0;
4405
4406         adev->in_suspend = true;
4407
4408         if (amdgpu_sriov_vf(adev)) {
4409                 amdgpu_virt_fini_data_exchange(adev);
4410                 r = amdgpu_virt_request_full_gpu(adev, false);
4411                 if (r)
4412                         return r;
4413         }
4414
4415         if (amdgpu_acpi_smart_shift_update(dev, AMDGPU_SS_DEV_D3))
4416                 DRM_WARN("smart shift update failed\n");
4417
4418         if (fbcon)
4419                 drm_fb_helper_set_suspend_unlocked(adev_to_drm(adev)->fb_helper, true);
4420
4421         cancel_delayed_work_sync(&adev->delayed_init_work);
4422         flush_delayed_work(&adev->gfx.gfx_off_delay_work);
4423
4424         amdgpu_ras_suspend(adev);
4425
4426         amdgpu_ttm_set_buffer_funcs_status(adev, false);
4427
4428         amdgpu_device_ip_suspend_phase1(adev);
4429
4430         if (!adev->in_s0ix)
4431                 amdgpu_amdkfd_suspend(adev, adev->in_runpm);
4432
4433         r = amdgpu_device_evict_resources(adev);
4434         if (r)
4435                 return r;
4436
4437         amdgpu_fence_driver_hw_fini(adev);
4438
4439         amdgpu_device_ip_suspend_phase2(adev);
4440
4441         if (amdgpu_sriov_vf(adev))
4442                 amdgpu_virt_release_full_gpu(adev, false);
4443
4444         return 0;
4445 }
4446
4447 /**
4448  * amdgpu_device_resume - initiate device resume
4449  *
4450  * @dev: drm dev pointer
4451  * @fbcon : notify the fbdev of resume
4452  *
4453  * Bring the hw back to operating state (all asics).
4454  * Returns 0 for success or an error on failure.
4455  * Called at driver resume.
4456  */
4457 int amdgpu_device_resume(struct drm_device *dev, bool fbcon)
4458 {
4459         struct amdgpu_device *adev = drm_to_adev(dev);
4460         int r = 0;
4461
4462         if (amdgpu_sriov_vf(adev)) {
4463                 r = amdgpu_virt_request_full_gpu(adev, true);
4464                 if (r)
4465                         return r;
4466         }
4467
4468         if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
4469                 return 0;
4470
4471         if (adev->in_s0ix)
4472                 amdgpu_dpm_gfx_state_change(adev, sGpuChangeState_D0Entry);
4473
4474         /* post card */
4475         if (amdgpu_device_need_post(adev)) {
4476                 r = amdgpu_device_asic_init(adev);
4477                 if (r)
4478                         dev_err(adev->dev, "amdgpu asic init failed\n");
4479         }
4480
4481         r = amdgpu_device_ip_resume(adev);
4482
4483         if (r) {
4484                 dev_err(adev->dev, "amdgpu_device_ip_resume failed (%d).\n", r);
4485                 goto exit;
4486         }
4487         amdgpu_fence_driver_hw_init(adev);
4488
4489         r = amdgpu_device_ip_late_init(adev);
4490         if (r)
4491                 goto exit;
4492
4493         queue_delayed_work(system_wq, &adev->delayed_init_work,
4494                            msecs_to_jiffies(AMDGPU_RESUME_MS));
4495
4496         if (!adev->in_s0ix) {
4497                 r = amdgpu_amdkfd_resume(adev, adev->in_runpm);
4498                 if (r)
4499                         goto exit;
4500         }
4501
4502 exit:
4503         if (amdgpu_sriov_vf(adev)) {
4504                 amdgpu_virt_init_data_exchange(adev);
4505                 amdgpu_virt_release_full_gpu(adev, true);
4506         }
4507
4508         if (r)
4509                 return r;
4510
4511         /* Make sure IB tests flushed */
4512         flush_delayed_work(&adev->delayed_init_work);
4513
4514         if (fbcon)
4515                 drm_fb_helper_set_suspend_unlocked(adev_to_drm(adev)->fb_helper, false);
4516
4517         amdgpu_ras_resume(adev);
4518
4519         if (adev->mode_info.num_crtc) {
4520                 /*
4521                  * Most of the connector probing functions try to acquire runtime pm
4522                  * refs to ensure that the GPU is powered on when connector polling is
4523                  * performed. Since we're calling this from a runtime PM callback,
4524                  * trying to acquire rpm refs will cause us to deadlock.
4525                  *
4526                  * Since we're guaranteed to be holding the rpm lock, it's safe to
4527                  * temporarily disable the rpm helpers so this doesn't deadlock us.
4528                  */
4529 #ifdef CONFIG_PM
4530                 dev->dev->power.disable_depth++;
4531 #endif
4532                 if (!adev->dc_enabled)
4533                         drm_helper_hpd_irq_event(dev);
4534                 else
4535                         drm_kms_helper_hotplug_event(dev);
4536 #ifdef CONFIG_PM
4537                 dev->dev->power.disable_depth--;
4538 #endif
4539         }
4540         adev->in_suspend = false;
4541
4542         if (adev->enable_mes)
4543                 amdgpu_mes_self_test(adev);
4544
4545         if (amdgpu_acpi_smart_shift_update(dev, AMDGPU_SS_DEV_D0))
4546                 DRM_WARN("smart shift update failed\n");
4547
4548         return 0;
4549 }
4550
4551 /**
4552  * amdgpu_device_ip_check_soft_reset - did soft reset succeed
4553  *
4554  * @adev: amdgpu_device pointer
4555  *
4556  * The list of all the hardware IPs that make up the asic is walked and
4557  * the check_soft_reset callbacks are run.  check_soft_reset determines
4558  * if the asic is still hung or not.
4559  * Returns true if any of the IPs are still in a hung state, false if not.
4560  */
4561 static bool amdgpu_device_ip_check_soft_reset(struct amdgpu_device *adev)
4562 {
4563         int i;
4564         bool asic_hang = false;
4565
4566         if (amdgpu_sriov_vf(adev))
4567                 return true;
4568
4569         if (amdgpu_asic_need_full_reset(adev))
4570                 return true;
4571
4572         for (i = 0; i < adev->num_ip_blocks; i++) {
4573                 if (!adev->ip_blocks[i].status.valid)
4574                         continue;
4575                 if (adev->ip_blocks[i].version->funcs->check_soft_reset)
4576                         adev->ip_blocks[i].status.hang =
4577                                 adev->ip_blocks[i].version->funcs->check_soft_reset(adev);
4578                 if (adev->ip_blocks[i].status.hang) {
4579                         dev_info(adev->dev, "IP block:%s is hung!\n", adev->ip_blocks[i].version->funcs->name);
4580                         asic_hang = true;
4581                 }
4582         }
4583         return asic_hang;
4584 }
4585
4586 /**
4587  * amdgpu_device_ip_pre_soft_reset - prepare for soft reset
4588  *
4589  * @adev: amdgpu_device pointer
4590  *
4591  * The list of all the hardware IPs that make up the asic is walked and the
4592  * pre_soft_reset callbacks are run if the block is hung.  pre_soft_reset
4593  * handles any IP specific hardware or software state changes that are
4594  * necessary for a soft reset to succeed.
4595  * Returns 0 on success, negative error code on failure.
4596  */
4597 static int amdgpu_device_ip_pre_soft_reset(struct amdgpu_device *adev)
4598 {
4599         int i, r = 0;
4600
4601         for (i = 0; i < adev->num_ip_blocks; i++) {
4602                 if (!adev->ip_blocks[i].status.valid)
4603                         continue;
4604                 if (adev->ip_blocks[i].status.hang &&
4605                     adev->ip_blocks[i].version->funcs->pre_soft_reset) {
4606                         r = adev->ip_blocks[i].version->funcs->pre_soft_reset(adev);
4607                         if (r)
4608                                 return r;
4609                 }
4610         }
4611
4612         return 0;
4613 }
4614
4615 /**
4616  * amdgpu_device_ip_need_full_reset - check if a full asic reset is needed
4617  *
4618  * @adev: amdgpu_device pointer
4619  *
4620  * Some hardware IPs cannot be soft reset.  If they are hung, a full gpu
4621  * reset is necessary to recover.
4622  * Returns true if a full asic reset is required, false if not.
4623  */
4624 static bool amdgpu_device_ip_need_full_reset(struct amdgpu_device *adev)
4625 {
4626         int i;
4627
4628         if (amdgpu_asic_need_full_reset(adev))
4629                 return true;
4630
4631         for (i = 0; i < adev->num_ip_blocks; i++) {
4632                 if (!adev->ip_blocks[i].status.valid)
4633                         continue;
4634                 if ((adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) ||
4635                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) ||
4636                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_ACP) ||
4637                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) ||
4638                      adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) {
4639                         if (adev->ip_blocks[i].status.hang) {
4640                                 dev_info(adev->dev, "Some block need full reset!\n");
4641                                 return true;
4642                         }
4643                 }
4644         }
4645         return false;
4646 }
4647
4648 /**
4649  * amdgpu_device_ip_soft_reset - do a soft reset
4650  *
4651  * @adev: amdgpu_device pointer
4652  *
4653  * The list of all the hardware IPs that make up the asic is walked and the
4654  * soft_reset callbacks are run if the block is hung.  soft_reset handles any
4655  * IP specific hardware or software state changes that are necessary to soft
4656  * reset the IP.
4657  * Returns 0 on success, negative error code on failure.
4658  */
4659 static int amdgpu_device_ip_soft_reset(struct amdgpu_device *adev)
4660 {
4661         int i, r = 0;
4662
4663         for (i = 0; i < adev->num_ip_blocks; i++) {
4664                 if (!adev->ip_blocks[i].status.valid)
4665                         continue;
4666                 if (adev->ip_blocks[i].status.hang &&
4667                     adev->ip_blocks[i].version->funcs->soft_reset) {
4668                         r = adev->ip_blocks[i].version->funcs->soft_reset(adev);
4669                         if (r)
4670                                 return r;
4671                 }
4672         }
4673
4674         return 0;
4675 }
4676
4677 /**
4678  * amdgpu_device_ip_post_soft_reset - clean up from soft reset
4679  *
4680  * @adev: amdgpu_device pointer
4681  *
4682  * The list of all the hardware IPs that make up the asic is walked and the
4683  * post_soft_reset callbacks are run if the asic was hung.  post_soft_reset
4684  * handles any IP specific hardware or software state changes that are
4685  * necessary after the IP has been soft reset.
4686  * Returns 0 on success, negative error code on failure.
4687  */
4688 static int amdgpu_device_ip_post_soft_reset(struct amdgpu_device *adev)
4689 {
4690         int i, r = 0;
4691
4692         for (i = 0; i < adev->num_ip_blocks; i++) {
4693                 if (!adev->ip_blocks[i].status.valid)
4694                         continue;
4695                 if (adev->ip_blocks[i].status.hang &&
4696                     adev->ip_blocks[i].version->funcs->post_soft_reset)
4697                         r = adev->ip_blocks[i].version->funcs->post_soft_reset(adev);
4698                 if (r)
4699                         return r;
4700         }
4701
4702         return 0;
4703 }
4704
4705 /**
4706  * amdgpu_device_recover_vram - Recover some VRAM contents
4707  *
4708  * @adev: amdgpu_device pointer
4709  *
4710  * Restores the contents of VRAM buffers from the shadows in GTT.  Used to
4711  * restore things like GPUVM page tables after a GPU reset where
4712  * the contents of VRAM might be lost.
4713  *
4714  * Returns:
4715  * 0 on success, negative error code on failure.
4716  */
4717 static int amdgpu_device_recover_vram(struct amdgpu_device *adev)
4718 {
4719         struct dma_fence *fence = NULL, *next = NULL;
4720         struct amdgpu_bo *shadow;
4721         struct amdgpu_bo_vm *vmbo;
4722         long r = 1, tmo;
4723
4724         if (amdgpu_sriov_runtime(adev))
4725                 tmo = msecs_to_jiffies(8000);
4726         else
4727                 tmo = msecs_to_jiffies(100);
4728
4729         dev_info(adev->dev, "recover vram bo from shadow start\n");
4730         mutex_lock(&adev->shadow_list_lock);
4731         list_for_each_entry(vmbo, &adev->shadow_list, shadow_list) {
4732                 /* If vm is compute context or adev is APU, shadow will be NULL */
4733                 if (!vmbo->shadow)
4734                         continue;
4735                 shadow = vmbo->shadow;
4736
4737                 /* No need to recover an evicted BO */
4738                 if (shadow->tbo.resource->mem_type != TTM_PL_TT ||
4739                     shadow->tbo.resource->start == AMDGPU_BO_INVALID_OFFSET ||
4740                     shadow->parent->tbo.resource->mem_type != TTM_PL_VRAM)
4741                         continue;
4742
4743                 r = amdgpu_bo_restore_shadow(shadow, &next);
4744                 if (r)
4745                         break;
4746
4747                 if (fence) {
4748                         tmo = dma_fence_wait_timeout(fence, false, tmo);
4749                         dma_fence_put(fence);
4750                         fence = next;
4751                         if (tmo == 0) {
4752                                 r = -ETIMEDOUT;
4753                                 break;
4754                         } else if (tmo < 0) {
4755                                 r = tmo;
4756                                 break;
4757                         }
4758                 } else {
4759                         fence = next;
4760                 }
4761         }
4762         mutex_unlock(&adev->shadow_list_lock);
4763
4764         if (fence)
4765                 tmo = dma_fence_wait_timeout(fence, false, tmo);
4766         dma_fence_put(fence);
4767
4768         if (r < 0 || tmo <= 0) {
4769                 dev_err(adev->dev, "recover vram bo from shadow failed, r is %ld, tmo is %ld\n", r, tmo);
4770                 return -EIO;
4771         }
4772
4773         dev_info(adev->dev, "recover vram bo from shadow done\n");
4774         return 0;
4775 }
4776
4777
4778 /**
4779  * amdgpu_device_reset_sriov - reset ASIC for SR-IOV vf
4780  *
4781  * @adev: amdgpu_device pointer
4782  * @from_hypervisor: request from hypervisor
4783  *
4784  * do VF FLR and reinitialize Asic
4785  * return 0 means succeeded otherwise failed
4786  */
4787 static int amdgpu_device_reset_sriov(struct amdgpu_device *adev,
4788                                      bool from_hypervisor)
4789 {
4790         int r;
4791         struct amdgpu_hive_info *hive = NULL;
4792         int retry_limit = 0;
4793
4794 retry:
4795         amdgpu_amdkfd_pre_reset(adev);
4796
4797         if (from_hypervisor)
4798                 r = amdgpu_virt_request_full_gpu(adev, true);
4799         else
4800                 r = amdgpu_virt_reset_gpu(adev);
4801         if (r)
4802                 return r;
4803         amdgpu_irq_gpu_reset_resume_helper(adev);
4804
4805         /* some sw clean up VF needs to do before recover */
4806         amdgpu_virt_post_reset(adev);
4807
4808         /* Resume IP prior to SMC */
4809         r = amdgpu_device_ip_reinit_early_sriov(adev);
4810         if (r)
4811                 goto error;
4812
4813         amdgpu_virt_init_data_exchange(adev);
4814
4815         r = amdgpu_device_fw_loading(adev);
4816         if (r)
4817                 return r;
4818
4819         /* now we are okay to resume SMC/CP/SDMA */
4820         r = amdgpu_device_ip_reinit_late_sriov(adev);
4821         if (r)
4822                 goto error;
4823
4824         hive = amdgpu_get_xgmi_hive(adev);
4825         /* Update PSP FW topology after reset */
4826         if (hive && adev->gmc.xgmi.num_physical_nodes > 1)
4827                 r = amdgpu_xgmi_update_topology(hive, adev);
4828
4829         if (hive)
4830                 amdgpu_put_xgmi_hive(hive);
4831
4832         if (!r) {
4833                 r = amdgpu_ib_ring_tests(adev);
4834
4835                 amdgpu_amdkfd_post_reset(adev);
4836         }
4837
4838 error:
4839         if (!r && adev->virt.gim_feature & AMDGIM_FEATURE_GIM_FLR_VRAMLOST) {
4840                 amdgpu_inc_vram_lost(adev);
4841                 r = amdgpu_device_recover_vram(adev);
4842         }
4843         amdgpu_virt_release_full_gpu(adev, true);
4844
4845         if (AMDGPU_RETRY_SRIOV_RESET(r)) {
4846                 if (retry_limit < AMDGPU_MAX_RETRY_LIMIT) {
4847                         retry_limit++;
4848                         goto retry;
4849                 } else
4850                         DRM_ERROR("GPU reset retry is beyond the retry limit\n");
4851         }
4852
4853         return r;
4854 }
4855
4856 /**
4857  * amdgpu_device_has_job_running - check if there is any job in mirror list
4858  *
4859  * @adev: amdgpu_device pointer
4860  *
4861  * check if there is any job in mirror list
4862  */
4863 bool amdgpu_device_has_job_running(struct amdgpu_device *adev)
4864 {
4865         int i;
4866         struct drm_sched_job *job;
4867
4868         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
4869                 struct amdgpu_ring *ring = adev->rings[i];
4870
4871                 if (!ring || !ring->sched.thread)
4872                         continue;
4873
4874                 spin_lock(&ring->sched.job_list_lock);
4875                 job = list_first_entry_or_null(&ring->sched.pending_list,
4876                                                struct drm_sched_job, list);
4877                 spin_unlock(&ring->sched.job_list_lock);
4878                 if (job)
4879                         return true;
4880         }
4881         return false;
4882 }
4883
4884 /**
4885  * amdgpu_device_should_recover_gpu - check if we should try GPU recovery
4886  *
4887  * @adev: amdgpu_device pointer
4888  *
4889  * Check amdgpu_gpu_recovery and SRIOV status to see if we should try to recover
4890  * a hung GPU.
4891  */
4892 bool amdgpu_device_should_recover_gpu(struct amdgpu_device *adev)
4893 {
4894
4895         if (amdgpu_gpu_recovery == 0)
4896                 goto disabled;
4897
4898         /* Skip soft reset check in fatal error mode */
4899         if (!amdgpu_ras_is_poison_mode_supported(adev))
4900                 return true;
4901
4902         if (amdgpu_sriov_vf(adev))
4903                 return true;
4904
4905         if (amdgpu_gpu_recovery == -1) {
4906                 switch (adev->asic_type) {
4907 #ifdef CONFIG_DRM_AMDGPU_SI
4908                 case CHIP_VERDE:
4909                 case CHIP_TAHITI:
4910                 case CHIP_PITCAIRN:
4911                 case CHIP_OLAND:
4912                 case CHIP_HAINAN:
4913 #endif
4914 #ifdef CONFIG_DRM_AMDGPU_CIK
4915                 case CHIP_KAVERI:
4916                 case CHIP_KABINI:
4917                 case CHIP_MULLINS:
4918 #endif
4919                 case CHIP_CARRIZO:
4920                 case CHIP_STONEY:
4921                 case CHIP_CYAN_SKILLFISH:
4922                         goto disabled;
4923                 default:
4924                         break;
4925                 }
4926         }
4927
4928         return true;
4929
4930 disabled:
4931                 dev_info(adev->dev, "GPU recovery disabled.\n");
4932                 return false;
4933 }
4934
4935 int amdgpu_device_mode1_reset(struct amdgpu_device *adev)
4936 {
4937         u32 i;
4938         int ret = 0;
4939
4940         amdgpu_atombios_scratch_regs_engine_hung(adev, true);
4941
4942         dev_info(adev->dev, "GPU mode1 reset\n");
4943
4944         /* disable BM */
4945         pci_clear_master(adev->pdev);
4946
4947         amdgpu_device_cache_pci_state(adev->pdev);
4948
4949         if (amdgpu_dpm_is_mode1_reset_supported(adev)) {
4950                 dev_info(adev->dev, "GPU smu mode1 reset\n");
4951                 ret = amdgpu_dpm_mode1_reset(adev);
4952         } else {
4953                 dev_info(adev->dev, "GPU psp mode1 reset\n");
4954                 ret = psp_gpu_reset(adev);
4955         }
4956
4957         if (ret)
4958                 goto mode1_reset_failed;
4959
4960         amdgpu_device_load_pci_state(adev->pdev);
4961         ret = amdgpu_psp_wait_for_bootloader(adev);
4962         if (ret)
4963                 goto mode1_reset_failed;
4964
4965         /* wait for asic to come out of reset */
4966         for (i = 0; i < adev->usec_timeout; i++) {
4967                 u32 memsize = adev->nbio.funcs->get_memsize(adev);
4968
4969                 if (memsize != 0xffffffff)
4970                         break;
4971                 udelay(1);
4972         }
4973
4974         if (i >= adev->usec_timeout) {
4975                 ret = -ETIMEDOUT;
4976                 goto mode1_reset_failed;
4977         }
4978
4979         amdgpu_atombios_scratch_regs_engine_hung(adev, false);
4980
4981         return 0;
4982
4983 mode1_reset_failed:
4984         dev_err(adev->dev, "GPU mode1 reset failed\n");
4985         return ret;
4986 }
4987
4988 int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev,
4989                                  struct amdgpu_reset_context *reset_context)
4990 {
4991         int i, r = 0;
4992         struct amdgpu_job *job = NULL;
4993         bool need_full_reset =
4994                 test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
4995
4996         if (reset_context->reset_req_dev == adev)
4997                 job = reset_context->job;
4998
4999         if (amdgpu_sriov_vf(adev)) {
5000                 /* stop the data exchange thread */
5001                 amdgpu_virt_fini_data_exchange(adev);
5002         }
5003
5004         amdgpu_fence_driver_isr_toggle(adev, true);
5005
5006         /* block all schedulers and reset given job's ring */
5007         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5008                 struct amdgpu_ring *ring = adev->rings[i];
5009
5010                 if (!ring || !ring->sched.thread)
5011                         continue;
5012
5013                 /* Clear job fence from fence drv to avoid force_completion
5014                  * leave NULL and vm flush fence in fence drv
5015                  */
5016                 amdgpu_fence_driver_clear_job_fences(ring);
5017
5018                 /* after all hw jobs are reset, hw fence is meaningless, so force_completion */
5019                 amdgpu_fence_driver_force_completion(ring);
5020         }
5021
5022         amdgpu_fence_driver_isr_toggle(adev, false);
5023
5024         if (job && job->vm)
5025                 drm_sched_increase_karma(&job->base);
5026
5027         r = amdgpu_reset_prepare_hwcontext(adev, reset_context);
5028         /* If reset handler not implemented, continue; otherwise return */
5029         if (r == -EOPNOTSUPP)
5030                 r = 0;
5031         else
5032                 return r;
5033
5034         /* Don't suspend on bare metal if we are not going to HW reset the ASIC */
5035         if (!amdgpu_sriov_vf(adev)) {
5036
5037                 if (!need_full_reset)
5038                         need_full_reset = amdgpu_device_ip_need_full_reset(adev);
5039
5040                 if (!need_full_reset && amdgpu_gpu_recovery &&
5041                     amdgpu_device_ip_check_soft_reset(adev)) {
5042                         amdgpu_device_ip_pre_soft_reset(adev);
5043                         r = amdgpu_device_ip_soft_reset(adev);
5044                         amdgpu_device_ip_post_soft_reset(adev);
5045                         if (r || amdgpu_device_ip_check_soft_reset(adev)) {
5046                                 dev_info(adev->dev, "soft reset failed, will fallback to full reset!\n");
5047                                 need_full_reset = true;
5048                         }
5049                 }
5050
5051                 if (need_full_reset)
5052                         r = amdgpu_device_ip_suspend(adev);
5053                 if (need_full_reset)
5054                         set_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5055                 else
5056                         clear_bit(AMDGPU_NEED_FULL_RESET,
5057                                   &reset_context->flags);
5058         }
5059
5060         return r;
5061 }
5062
5063 static int amdgpu_reset_reg_dumps(struct amdgpu_device *adev)
5064 {
5065         int i;
5066
5067         lockdep_assert_held(&adev->reset_domain->sem);
5068
5069         for (i = 0; i < adev->reset_info.num_regs; i++) {
5070                 adev->reset_info.reset_dump_reg_value[i] =
5071                         RREG32(adev->reset_info.reset_dump_reg_list[i]);
5072
5073                 trace_amdgpu_reset_reg_dumps(adev->reset_info.reset_dump_reg_list[i],
5074                                              adev->reset_info.reset_dump_reg_value[i]);
5075         }
5076
5077         return 0;
5078 }
5079
5080 int amdgpu_do_asic_reset(struct list_head *device_list_handle,
5081                          struct amdgpu_reset_context *reset_context)
5082 {
5083         struct amdgpu_device *tmp_adev = NULL;
5084         bool need_full_reset, skip_hw_reset, vram_lost = false;
5085         int r = 0;
5086         bool gpu_reset_for_dev_remove = 0;
5087
5088         /* Try reset handler method first */
5089         tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device,
5090                                     reset_list);
5091         amdgpu_reset_reg_dumps(tmp_adev);
5092
5093         reset_context->reset_device_list = device_list_handle;
5094         r = amdgpu_reset_perform_reset(tmp_adev, reset_context);
5095         /* If reset handler not implemented, continue; otherwise return */
5096         if (r == -EOPNOTSUPP)
5097                 r = 0;
5098         else
5099                 return r;
5100
5101         /* Reset handler not implemented, use the default method */
5102         need_full_reset =
5103                 test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5104         skip_hw_reset = test_bit(AMDGPU_SKIP_HW_RESET, &reset_context->flags);
5105
5106         gpu_reset_for_dev_remove =
5107                 test_bit(AMDGPU_RESET_FOR_DEVICE_REMOVE, &reset_context->flags) &&
5108                         test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5109
5110         /*
5111          * ASIC reset has to be done on all XGMI hive nodes ASAP
5112          * to allow proper links negotiation in FW (within 1 sec)
5113          */
5114         if (!skip_hw_reset && need_full_reset) {
5115                 list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5116                         /* For XGMI run all resets in parallel to speed up the process */
5117                         if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) {
5118                                 tmp_adev->gmc.xgmi.pending_reset = false;
5119                                 if (!queue_work(system_unbound_wq, &tmp_adev->xgmi_reset_work))
5120                                         r = -EALREADY;
5121                         } else
5122                                 r = amdgpu_asic_reset(tmp_adev);
5123
5124                         if (r) {
5125                                 dev_err(tmp_adev->dev, "ASIC reset failed with error, %d for drm dev, %s",
5126                                          r, adev_to_drm(tmp_adev)->unique);
5127                                 goto out;
5128                         }
5129                 }
5130
5131                 /* For XGMI wait for all resets to complete before proceed */
5132                 if (!r) {
5133                         list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5134                                 if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) {
5135                                         flush_work(&tmp_adev->xgmi_reset_work);
5136                                         r = tmp_adev->asic_reset_res;
5137                                         if (r)
5138                                                 break;
5139                                 }
5140                         }
5141                 }
5142         }
5143
5144         if (!r && amdgpu_ras_intr_triggered()) {
5145                 list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5146                         amdgpu_ras_reset_error_count(tmp_adev, AMDGPU_RAS_BLOCK__MMHUB);
5147                 }
5148
5149                 amdgpu_ras_intr_cleared();
5150         }
5151
5152         /* Since the mode1 reset affects base ip blocks, the
5153          * phase1 ip blocks need to be resumed. Otherwise there
5154          * will be a BIOS signature error and the psp bootloader
5155          * can't load kdb on the next amdgpu install.
5156          */
5157         if (gpu_reset_for_dev_remove) {
5158                 list_for_each_entry(tmp_adev, device_list_handle, reset_list)
5159                         amdgpu_device_ip_resume_phase1(tmp_adev);
5160
5161                 goto end;
5162         }
5163
5164         list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5165                 if (need_full_reset) {
5166                         /* post card */
5167                         r = amdgpu_device_asic_init(tmp_adev);
5168                         if (r) {
5169                                 dev_warn(tmp_adev->dev, "asic atom init failed!");
5170                         } else {
5171                                 dev_info(tmp_adev->dev, "GPU reset succeeded, trying to resume\n");
5172
5173                                 r = amdgpu_device_ip_resume_phase1(tmp_adev);
5174                                 if (r)
5175                                         goto out;
5176
5177                                 vram_lost = amdgpu_device_check_vram_lost(tmp_adev);
5178
5179                                 amdgpu_coredump(tmp_adev, vram_lost, reset_context);
5180
5181                                 if (vram_lost) {
5182                                         DRM_INFO("VRAM is lost due to GPU reset!\n");
5183                                         amdgpu_inc_vram_lost(tmp_adev);
5184                                 }
5185
5186                                 r = amdgpu_device_fw_loading(tmp_adev);
5187                                 if (r)
5188                                         return r;
5189
5190                                 r = amdgpu_xcp_restore_partition_mode(
5191                                         tmp_adev->xcp_mgr);
5192                                 if (r)
5193                                         goto out;
5194
5195                                 r = amdgpu_device_ip_resume_phase2(tmp_adev);
5196                                 if (r)
5197                                         goto out;
5198
5199                                 if (tmp_adev->mman.buffer_funcs_ring->sched.ready)
5200                                         amdgpu_ttm_set_buffer_funcs_status(tmp_adev, true);
5201
5202                                 if (vram_lost)
5203                                         amdgpu_device_fill_reset_magic(tmp_adev);
5204
5205                                 /*
5206                                  * Add this ASIC as tracked as reset was already
5207                                  * complete successfully.
5208                                  */
5209                                 amdgpu_register_gpu_instance(tmp_adev);
5210
5211                                 if (!reset_context->hive &&
5212                                     tmp_adev->gmc.xgmi.num_physical_nodes > 1)
5213                                         amdgpu_xgmi_add_device(tmp_adev);
5214
5215                                 r = amdgpu_device_ip_late_init(tmp_adev);
5216                                 if (r)
5217                                         goto out;
5218
5219                                 drm_fb_helper_set_suspend_unlocked(adev_to_drm(tmp_adev)->fb_helper, false);
5220
5221                                 /*
5222                                  * The GPU enters bad state once faulty pages
5223                                  * by ECC has reached the threshold, and ras
5224                                  * recovery is scheduled next. So add one check
5225                                  * here to break recovery if it indeed exceeds
5226                                  * bad page threshold, and remind user to
5227                                  * retire this GPU or setting one bigger
5228                                  * bad_page_threshold value to fix this once
5229                                  * probing driver again.
5230                                  */
5231                                 if (!amdgpu_ras_eeprom_check_err_threshold(tmp_adev)) {
5232                                         /* must succeed. */
5233                                         amdgpu_ras_resume(tmp_adev);
5234                                 } else {
5235                                         r = -EINVAL;
5236                                         goto out;
5237                                 }
5238
5239                                 /* Update PSP FW topology after reset */
5240                                 if (reset_context->hive &&
5241                                     tmp_adev->gmc.xgmi.num_physical_nodes > 1)
5242                                         r = amdgpu_xgmi_update_topology(
5243                                                 reset_context->hive, tmp_adev);
5244                         }
5245                 }
5246
5247 out:
5248                 if (!r) {
5249                         amdgpu_irq_gpu_reset_resume_helper(tmp_adev);
5250                         r = amdgpu_ib_ring_tests(tmp_adev);
5251                         if (r) {
5252                                 dev_err(tmp_adev->dev, "ib ring test failed (%d).\n", r);
5253                                 need_full_reset = true;
5254                                 r = -EAGAIN;
5255                                 goto end;
5256                         }
5257                 }
5258
5259                 if (!r)
5260                         r = amdgpu_device_recover_vram(tmp_adev);
5261                 else
5262                         tmp_adev->asic_reset_res = r;
5263         }
5264
5265 end:
5266         if (need_full_reset)
5267                 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5268         else
5269                 clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5270         return r;
5271 }
5272
5273 static void amdgpu_device_set_mp1_state(struct amdgpu_device *adev)
5274 {
5275
5276         switch (amdgpu_asic_reset_method(adev)) {
5277         case AMD_RESET_METHOD_MODE1:
5278                 adev->mp1_state = PP_MP1_STATE_SHUTDOWN;
5279                 break;
5280         case AMD_RESET_METHOD_MODE2:
5281                 adev->mp1_state = PP_MP1_STATE_RESET;
5282                 break;
5283         default:
5284                 adev->mp1_state = PP_MP1_STATE_NONE;
5285                 break;
5286         }
5287 }
5288
5289 static void amdgpu_device_unset_mp1_state(struct amdgpu_device *adev)
5290 {
5291         amdgpu_vf_error_trans_all(adev);
5292         adev->mp1_state = PP_MP1_STATE_NONE;
5293 }
5294
5295 static void amdgpu_device_resume_display_audio(struct amdgpu_device *adev)
5296 {
5297         struct pci_dev *p = NULL;
5298
5299         p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus),
5300                         adev->pdev->bus->number, 1);
5301         if (p) {
5302                 pm_runtime_enable(&(p->dev));
5303                 pm_runtime_resume(&(p->dev));
5304         }
5305
5306         pci_dev_put(p);
5307 }
5308
5309 static int amdgpu_device_suspend_display_audio(struct amdgpu_device *adev)
5310 {
5311         enum amd_reset_method reset_method;
5312         struct pci_dev *p = NULL;
5313         u64 expires;
5314
5315         /*
5316          * For now, only BACO and mode1 reset are confirmed
5317          * to suffer the audio issue without proper suspended.
5318          */
5319         reset_method = amdgpu_asic_reset_method(adev);
5320         if ((reset_method != AMD_RESET_METHOD_BACO) &&
5321              (reset_method != AMD_RESET_METHOD_MODE1))
5322                 return -EINVAL;
5323
5324         p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus),
5325                         adev->pdev->bus->number, 1);
5326         if (!p)
5327                 return -ENODEV;
5328
5329         expires = pm_runtime_autosuspend_expiration(&(p->dev));
5330         if (!expires)
5331                 /*
5332                  * If we cannot get the audio device autosuspend delay,
5333                  * a fixed 4S interval will be used. Considering 3S is
5334                  * the audio controller default autosuspend delay setting.
5335                  * 4S used here is guaranteed to cover that.
5336                  */
5337                 expires = ktime_get_mono_fast_ns() + NSEC_PER_SEC * 4ULL;
5338
5339         while (!pm_runtime_status_suspended(&(p->dev))) {
5340                 if (!pm_runtime_suspend(&(p->dev)))
5341                         break;
5342
5343                 if (expires < ktime_get_mono_fast_ns()) {
5344                         dev_warn(adev->dev, "failed to suspend display audio\n");
5345                         pci_dev_put(p);
5346                         /* TODO: abort the succeeding gpu reset? */
5347                         return -ETIMEDOUT;
5348                 }
5349         }
5350
5351         pm_runtime_disable(&(p->dev));
5352
5353         pci_dev_put(p);
5354         return 0;
5355 }
5356
5357 static inline void amdgpu_device_stop_pending_resets(struct amdgpu_device *adev)
5358 {
5359         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
5360
5361 #if defined(CONFIG_DEBUG_FS)
5362         if (!amdgpu_sriov_vf(adev))
5363                 cancel_work(&adev->reset_work);
5364 #endif
5365
5366         if (adev->kfd.dev)
5367                 cancel_work(&adev->kfd.reset_work);
5368
5369         if (amdgpu_sriov_vf(adev))
5370                 cancel_work(&adev->virt.flr_work);
5371
5372         if (con && adev->ras_enabled)
5373                 cancel_work(&con->recovery_work);
5374
5375 }
5376
5377 /**
5378  * amdgpu_device_gpu_recover - reset the asic and recover scheduler
5379  *
5380  * @adev: amdgpu_device pointer
5381  * @job: which job trigger hang
5382  * @reset_context: amdgpu reset context pointer
5383  *
5384  * Attempt to reset the GPU if it has hung (all asics).
5385  * Attempt to do soft-reset or full-reset and reinitialize Asic
5386  * Returns 0 for success or an error on failure.
5387  */
5388
5389 int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
5390                               struct amdgpu_job *job,
5391                               struct amdgpu_reset_context *reset_context)
5392 {
5393         struct list_head device_list, *device_list_handle =  NULL;
5394         bool job_signaled = false;
5395         struct amdgpu_hive_info *hive = NULL;
5396         struct amdgpu_device *tmp_adev = NULL;
5397         int i, r = 0;
5398         bool need_emergency_restart = false;
5399         bool audio_suspended = false;
5400         bool gpu_reset_for_dev_remove = false;
5401
5402         gpu_reset_for_dev_remove =
5403                         test_bit(AMDGPU_RESET_FOR_DEVICE_REMOVE, &reset_context->flags) &&
5404                                 test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5405
5406         /*
5407          * Special case: RAS triggered and full reset isn't supported
5408          */
5409         need_emergency_restart = amdgpu_ras_need_emergency_restart(adev);
5410
5411         /*
5412          * Flush RAM to disk so that after reboot
5413          * the user can read log and see why the system rebooted.
5414          */
5415         if (need_emergency_restart && amdgpu_ras_get_context(adev) &&
5416                 amdgpu_ras_get_context(adev)->reboot) {
5417                 DRM_WARN("Emergency reboot.");
5418
5419                 ksys_sync_helper();
5420                 emergency_restart();
5421         }
5422
5423         dev_info(adev->dev, "GPU %s begin!\n",
5424                 need_emergency_restart ? "jobs stop":"reset");
5425
5426         if (!amdgpu_sriov_vf(adev))
5427                 hive = amdgpu_get_xgmi_hive(adev);
5428         if (hive)
5429                 mutex_lock(&hive->hive_lock);
5430
5431         reset_context->job = job;
5432         reset_context->hive = hive;
5433         /*
5434          * Build list of devices to reset.
5435          * In case we are in XGMI hive mode, resort the device list
5436          * to put adev in the 1st position.
5437          */
5438         INIT_LIST_HEAD(&device_list);
5439         if (!amdgpu_sriov_vf(adev) && (adev->gmc.xgmi.num_physical_nodes > 1)) {
5440                 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
5441                         list_add_tail(&tmp_adev->reset_list, &device_list);
5442                         if (gpu_reset_for_dev_remove && adev->shutdown)
5443                                 tmp_adev->shutdown = true;
5444                 }
5445                 if (!list_is_first(&adev->reset_list, &device_list))
5446                         list_rotate_to_front(&adev->reset_list, &device_list);
5447                 device_list_handle = &device_list;
5448         } else {
5449                 list_add_tail(&adev->reset_list, &device_list);
5450                 device_list_handle = &device_list;
5451         }
5452
5453         /* We need to lock reset domain only once both for XGMI and single device */
5454         tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device,
5455                                     reset_list);
5456         amdgpu_device_lock_reset_domain(tmp_adev->reset_domain);
5457
5458         /* block all schedulers and reset given job's ring */
5459         list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5460
5461                 amdgpu_device_set_mp1_state(tmp_adev);
5462
5463                 /*
5464                  * Try to put the audio codec into suspend state
5465                  * before gpu reset started.
5466                  *
5467                  * Due to the power domain of the graphics device
5468                  * is shared with AZ power domain. Without this,
5469                  * we may change the audio hardware from behind
5470                  * the audio driver's back. That will trigger
5471                  * some audio codec errors.
5472                  */
5473                 if (!amdgpu_device_suspend_display_audio(tmp_adev))
5474                         audio_suspended = true;
5475
5476                 amdgpu_ras_set_error_query_ready(tmp_adev, false);
5477
5478                 cancel_delayed_work_sync(&tmp_adev->delayed_init_work);
5479
5480                 if (!amdgpu_sriov_vf(tmp_adev))
5481                         amdgpu_amdkfd_pre_reset(tmp_adev);
5482
5483                 /*
5484                  * Mark these ASICs to be reseted as untracked first
5485                  * And add them back after reset completed
5486                  */
5487                 amdgpu_unregister_gpu_instance(tmp_adev);
5488
5489                 drm_fb_helper_set_suspend_unlocked(adev_to_drm(tmp_adev)->fb_helper, true);
5490
5491                 /* disable ras on ALL IPs */
5492                 if (!need_emergency_restart &&
5493                       amdgpu_device_ip_need_full_reset(tmp_adev))
5494                         amdgpu_ras_suspend(tmp_adev);
5495
5496                 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5497                         struct amdgpu_ring *ring = tmp_adev->rings[i];
5498
5499                         if (!ring || !ring->sched.thread)
5500                                 continue;
5501
5502                         drm_sched_stop(&ring->sched, job ? &job->base : NULL);
5503
5504                         if (need_emergency_restart)
5505                                 amdgpu_job_stop_all_jobs_on_sched(&ring->sched);
5506                 }
5507                 atomic_inc(&tmp_adev->gpu_reset_counter);
5508         }
5509
5510         if (need_emergency_restart)
5511                 goto skip_sched_resume;
5512
5513         /*
5514          * Must check guilty signal here since after this point all old
5515          * HW fences are force signaled.
5516          *
5517          * job->base holds a reference to parent fence
5518          */
5519         if (job && dma_fence_is_signaled(&job->hw_fence)) {
5520                 job_signaled = true;
5521                 dev_info(adev->dev, "Guilty job already signaled, skipping HW reset");
5522                 goto skip_hw_reset;
5523         }
5524
5525 retry:  /* Rest of adevs pre asic reset from XGMI hive. */
5526         list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5527                 if (gpu_reset_for_dev_remove) {
5528                         /* Workaroud for ASICs need to disable SMC first */
5529                         amdgpu_device_smu_fini_early(tmp_adev);
5530                 }
5531                 r = amdgpu_device_pre_asic_reset(tmp_adev, reset_context);
5532                 /*TODO Should we stop ?*/
5533                 if (r) {
5534                         dev_err(tmp_adev->dev, "GPU pre asic reset failed with err, %d for drm dev, %s ",
5535                                   r, adev_to_drm(tmp_adev)->unique);
5536                         tmp_adev->asic_reset_res = r;
5537                 }
5538
5539                 /*
5540                  * Drop all pending non scheduler resets. Scheduler resets
5541                  * were already dropped during drm_sched_stop
5542                  */
5543                 amdgpu_device_stop_pending_resets(tmp_adev);
5544         }
5545
5546         /* Actual ASIC resets if needed.*/
5547         /* Host driver will handle XGMI hive reset for SRIOV */
5548         if (amdgpu_sriov_vf(adev)) {
5549                 r = amdgpu_device_reset_sriov(adev, job ? false : true);
5550                 if (r)
5551                         adev->asic_reset_res = r;
5552
5553                 /* Aldebaran and gfx_11_0_3 support ras in SRIOV, so need resume ras during reset */
5554                 if (amdgpu_ip_version(adev, GC_HWIP, 0) ==
5555                             IP_VERSION(9, 4, 2) ||
5556                     amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 0, 3))
5557                         amdgpu_ras_resume(adev);
5558         } else {
5559                 r = amdgpu_do_asic_reset(device_list_handle, reset_context);
5560                 if (r && r == -EAGAIN)
5561                         goto retry;
5562
5563                 if (!r && gpu_reset_for_dev_remove)
5564                         goto recover_end;
5565         }
5566
5567 skip_hw_reset:
5568
5569         /* Post ASIC reset for all devs .*/
5570         list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5571
5572                 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5573                         struct amdgpu_ring *ring = tmp_adev->rings[i];
5574
5575                         if (!ring || !ring->sched.thread)
5576                                 continue;
5577
5578                         drm_sched_start(&ring->sched, true);
5579                 }
5580
5581                 if (!drm_drv_uses_atomic_modeset(adev_to_drm(tmp_adev)) && !job_signaled)
5582                         drm_helper_resume_force_mode(adev_to_drm(tmp_adev));
5583
5584                 if (tmp_adev->asic_reset_res)
5585                         r = tmp_adev->asic_reset_res;
5586
5587                 tmp_adev->asic_reset_res = 0;
5588
5589                 if (r) {
5590                         /* bad news, how to tell it to userspace ? */
5591                         dev_info(tmp_adev->dev, "GPU reset(%d) failed\n", atomic_read(&tmp_adev->gpu_reset_counter));
5592                         amdgpu_vf_error_put(tmp_adev, AMDGIM_ERROR_VF_GPU_RESET_FAIL, 0, r);
5593                 } else {
5594                         dev_info(tmp_adev->dev, "GPU reset(%d) succeeded!\n", atomic_read(&tmp_adev->gpu_reset_counter));
5595                         if (amdgpu_acpi_smart_shift_update(adev_to_drm(tmp_adev), AMDGPU_SS_DEV_D0))
5596                                 DRM_WARN("smart shift update failed\n");
5597                 }
5598         }
5599
5600 skip_sched_resume:
5601         list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5602                 /* unlock kfd: SRIOV would do it separately */
5603                 if (!need_emergency_restart && !amdgpu_sriov_vf(tmp_adev))
5604                         amdgpu_amdkfd_post_reset(tmp_adev);
5605
5606                 /* kfd_post_reset will do nothing if kfd device is not initialized,
5607                  * need to bring up kfd here if it's not be initialized before
5608                  */
5609                 if (!adev->kfd.init_complete)
5610                         amdgpu_amdkfd_device_init(adev);
5611
5612                 if (audio_suspended)
5613                         amdgpu_device_resume_display_audio(tmp_adev);
5614
5615                 amdgpu_device_unset_mp1_state(tmp_adev);
5616
5617                 amdgpu_ras_set_error_query_ready(tmp_adev, true);
5618         }
5619
5620 recover_end:
5621         tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device,
5622                                             reset_list);
5623         amdgpu_device_unlock_reset_domain(tmp_adev->reset_domain);
5624
5625         if (hive) {
5626                 mutex_unlock(&hive->hive_lock);
5627                 amdgpu_put_xgmi_hive(hive);
5628         }
5629
5630         if (r)
5631                 dev_info(adev->dev, "GPU reset end with ret = %d\n", r);
5632
5633         atomic_set(&adev->reset_domain->reset_res, r);
5634         return r;
5635 }
5636
5637 /**
5638  * amdgpu_device_get_pcie_info - fence pcie info about the PCIE slot
5639  *
5640  * @adev: amdgpu_device pointer
5641  *
5642  * Fetchs and stores in the driver the PCIE capabilities (gen speed
5643  * and lanes) of the slot the device is in. Handles APUs and
5644  * virtualized environments where PCIE config space may not be available.
5645  */
5646 static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev)
5647 {
5648         struct pci_dev *pdev;
5649         enum pci_bus_speed speed_cap, platform_speed_cap;
5650         enum pcie_link_width platform_link_width;
5651
5652         if (amdgpu_pcie_gen_cap)
5653                 adev->pm.pcie_gen_mask = amdgpu_pcie_gen_cap;
5654
5655         if (amdgpu_pcie_lane_cap)
5656                 adev->pm.pcie_mlw_mask = amdgpu_pcie_lane_cap;
5657
5658         /* covers APUs as well */
5659         if (pci_is_root_bus(adev->pdev->bus) && !amdgpu_passthrough(adev)) {
5660                 if (adev->pm.pcie_gen_mask == 0)
5661                         adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK;
5662                 if (adev->pm.pcie_mlw_mask == 0)
5663                         adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK;
5664                 return;
5665         }
5666
5667         if (adev->pm.pcie_gen_mask && adev->pm.pcie_mlw_mask)
5668                 return;
5669
5670         pcie_bandwidth_available(adev->pdev, NULL,
5671                                  &platform_speed_cap, &platform_link_width);
5672
5673         if (adev->pm.pcie_gen_mask == 0) {
5674                 /* asic caps */
5675                 pdev = adev->pdev;
5676                 speed_cap = pcie_get_speed_cap(pdev);
5677                 if (speed_cap == PCI_SPEED_UNKNOWN) {
5678                         adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5679                                                   CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5680                                                   CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
5681                 } else {
5682                         if (speed_cap == PCIE_SPEED_32_0GT)
5683                                 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5684                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5685                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5686                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4 |
5687                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN5);
5688                         else if (speed_cap == PCIE_SPEED_16_0GT)
5689                                 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5690                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5691                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5692                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4);
5693                         else if (speed_cap == PCIE_SPEED_8_0GT)
5694                                 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5695                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5696                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
5697                         else if (speed_cap == PCIE_SPEED_5_0GT)
5698                                 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5699                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2);
5700                         else
5701                                 adev->pm.pcie_gen_mask |= CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1;
5702                 }
5703                 /* platform caps */
5704                 if (platform_speed_cap == PCI_SPEED_UNKNOWN) {
5705                         adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5706                                                    CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2);
5707                 } else {
5708                         if (platform_speed_cap == PCIE_SPEED_32_0GT)
5709                                 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5710                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5711                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5712                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4 |
5713                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN5);
5714                         else if (platform_speed_cap == PCIE_SPEED_16_0GT)
5715                                 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5716                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5717                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5718                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4);
5719                         else if (platform_speed_cap == PCIE_SPEED_8_0GT)
5720                                 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5721                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5722                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3);
5723                         else if (platform_speed_cap == PCIE_SPEED_5_0GT)
5724                                 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5725                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2);
5726                         else
5727                                 adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1;
5728
5729                 }
5730         }
5731         if (adev->pm.pcie_mlw_mask == 0) {
5732                 if (platform_link_width == PCIE_LNK_WIDTH_UNKNOWN) {
5733                         adev->pm.pcie_mlw_mask |= AMDGPU_DEFAULT_PCIE_MLW_MASK;
5734                 } else {
5735                         switch (platform_link_width) {
5736                         case PCIE_LNK_X32:
5737                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X32 |
5738                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
5739                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
5740                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5741                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5742                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5743                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5744                                 break;
5745                         case PCIE_LNK_X16:
5746                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
5747                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
5748                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5749                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5750                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5751                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5752                                 break;
5753                         case PCIE_LNK_X12:
5754                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
5755                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5756                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5757                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5758                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5759                                 break;
5760                         case PCIE_LNK_X8:
5761                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5762                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5763                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5764                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5765                                 break;
5766                         case PCIE_LNK_X4:
5767                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5768                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5769                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5770                                 break;
5771                         case PCIE_LNK_X2:
5772                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5773                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5774                                 break;
5775                         case PCIE_LNK_X1:
5776                                 adev->pm.pcie_mlw_mask = CAIL_PCIE_LINK_WIDTH_SUPPORT_X1;
5777                                 break;
5778                         default:
5779                                 break;
5780                         }
5781                 }
5782         }
5783 }
5784
5785 /**
5786  * amdgpu_device_is_peer_accessible - Check peer access through PCIe BAR
5787  *
5788  * @adev: amdgpu_device pointer
5789  * @peer_adev: amdgpu_device pointer for peer device trying to access @adev
5790  *
5791  * Return true if @peer_adev can access (DMA) @adev through the PCIe
5792  * BAR, i.e. @adev is "large BAR" and the BAR matches the DMA mask of
5793  * @peer_adev.
5794  */
5795 bool amdgpu_device_is_peer_accessible(struct amdgpu_device *adev,
5796                                       struct amdgpu_device *peer_adev)
5797 {
5798 #ifdef CONFIG_HSA_AMD_P2P
5799         uint64_t address_mask = peer_adev->dev->dma_mask ?
5800                 ~*peer_adev->dev->dma_mask : ~((1ULL << 32) - 1);
5801         resource_size_t aper_limit =
5802                 adev->gmc.aper_base + adev->gmc.aper_size - 1;
5803         bool p2p_access =
5804                 !adev->gmc.xgmi.connected_to_cpu &&
5805                 !(pci_p2pdma_distance(adev->pdev, peer_adev->dev, false) < 0);
5806
5807         return pcie_p2p && p2p_access && (adev->gmc.visible_vram_size &&
5808                 adev->gmc.real_vram_size == adev->gmc.visible_vram_size &&
5809                 !(adev->gmc.aper_base & address_mask ||
5810                   aper_limit & address_mask));
5811 #else
5812         return false;
5813 #endif
5814 }
5815
5816 int amdgpu_device_baco_enter(struct drm_device *dev)
5817 {
5818         struct amdgpu_device *adev = drm_to_adev(dev);
5819         struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
5820
5821         if (!amdgpu_device_supports_baco(dev))
5822                 return -ENOTSUPP;
5823
5824         if (ras && adev->ras_enabled &&
5825             adev->nbio.funcs->enable_doorbell_interrupt)
5826                 adev->nbio.funcs->enable_doorbell_interrupt(adev, false);
5827
5828         return amdgpu_dpm_baco_enter(adev);
5829 }
5830
5831 int amdgpu_device_baco_exit(struct drm_device *dev)
5832 {
5833         struct amdgpu_device *adev = drm_to_adev(dev);
5834         struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
5835         int ret = 0;
5836
5837         if (!amdgpu_device_supports_baco(dev))
5838                 return -ENOTSUPP;
5839
5840         ret = amdgpu_dpm_baco_exit(adev);
5841         if (ret)
5842                 return ret;
5843
5844         if (ras && adev->ras_enabled &&
5845             adev->nbio.funcs->enable_doorbell_interrupt)
5846                 adev->nbio.funcs->enable_doorbell_interrupt(adev, true);
5847
5848         if (amdgpu_passthrough(adev) &&
5849             adev->nbio.funcs->clear_doorbell_interrupt)
5850                 adev->nbio.funcs->clear_doorbell_interrupt(adev);
5851
5852         return 0;
5853 }
5854
5855 /**
5856  * amdgpu_pci_error_detected - Called when a PCI error is detected.
5857  * @pdev: PCI device struct
5858  * @state: PCI channel state
5859  *
5860  * Description: Called when a PCI error is detected.
5861  *
5862  * Return: PCI_ERS_RESULT_NEED_RESET or PCI_ERS_RESULT_DISCONNECT.
5863  */
5864 pci_ers_result_t amdgpu_pci_error_detected(struct pci_dev *pdev, pci_channel_state_t state)
5865 {
5866         struct drm_device *dev = pci_get_drvdata(pdev);
5867         struct amdgpu_device *adev = drm_to_adev(dev);
5868         int i;
5869
5870         DRM_INFO("PCI error: detected callback, state(%d)!!\n", state);
5871
5872         if (adev->gmc.xgmi.num_physical_nodes > 1) {
5873                 DRM_WARN("No support for XGMI hive yet...");
5874                 return PCI_ERS_RESULT_DISCONNECT;
5875         }
5876
5877         adev->pci_channel_state = state;
5878
5879         switch (state) {
5880         case pci_channel_io_normal:
5881                 return PCI_ERS_RESULT_CAN_RECOVER;
5882         /* Fatal error, prepare for slot reset */
5883         case pci_channel_io_frozen:
5884                 /*
5885                  * Locking adev->reset_domain->sem will prevent any external access
5886                  * to GPU during PCI error recovery
5887                  */
5888                 amdgpu_device_lock_reset_domain(adev->reset_domain);
5889                 amdgpu_device_set_mp1_state(adev);
5890
5891                 /*
5892                  * Block any work scheduling as we do for regular GPU reset
5893                  * for the duration of the recovery
5894                  */
5895                 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5896                         struct amdgpu_ring *ring = adev->rings[i];
5897
5898                         if (!ring || !ring->sched.thread)
5899                                 continue;
5900
5901                         drm_sched_stop(&ring->sched, NULL);
5902                 }
5903                 atomic_inc(&adev->gpu_reset_counter);
5904                 return PCI_ERS_RESULT_NEED_RESET;
5905         case pci_channel_io_perm_failure:
5906                 /* Permanent error, prepare for device removal */
5907                 return PCI_ERS_RESULT_DISCONNECT;
5908         }
5909
5910         return PCI_ERS_RESULT_NEED_RESET;
5911 }
5912
5913 /**
5914  * amdgpu_pci_mmio_enabled - Enable MMIO and dump debug registers
5915  * @pdev: pointer to PCI device
5916  */
5917 pci_ers_result_t amdgpu_pci_mmio_enabled(struct pci_dev *pdev)
5918 {
5919
5920         DRM_INFO("PCI error: mmio enabled callback!!\n");
5921
5922         /* TODO - dump whatever for debugging purposes */
5923
5924         /* This called only if amdgpu_pci_error_detected returns
5925          * PCI_ERS_RESULT_CAN_RECOVER. Read/write to the device still
5926          * works, no need to reset slot.
5927          */
5928
5929         return PCI_ERS_RESULT_RECOVERED;
5930 }
5931
5932 /**
5933  * amdgpu_pci_slot_reset - Called when PCI slot has been reset.
5934  * @pdev: PCI device struct
5935  *
5936  * Description: This routine is called by the pci error recovery
5937  * code after the PCI slot has been reset, just before we
5938  * should resume normal operations.
5939  */
5940 pci_ers_result_t amdgpu_pci_slot_reset(struct pci_dev *pdev)
5941 {
5942         struct drm_device *dev = pci_get_drvdata(pdev);
5943         struct amdgpu_device *adev = drm_to_adev(dev);
5944         int r, i;
5945         struct amdgpu_reset_context reset_context;
5946         u32 memsize;
5947         struct list_head device_list;
5948
5949         DRM_INFO("PCI error: slot reset callback!!\n");
5950
5951         memset(&reset_context, 0, sizeof(reset_context));
5952
5953         INIT_LIST_HEAD(&device_list);
5954         list_add_tail(&adev->reset_list, &device_list);
5955
5956         /* wait for asic to come out of reset */
5957         msleep(500);
5958
5959         /* Restore PCI confspace */
5960         amdgpu_device_load_pci_state(pdev);
5961
5962         /* confirm  ASIC came out of reset */
5963         for (i = 0; i < adev->usec_timeout; i++) {
5964                 memsize = amdgpu_asic_get_config_memsize(adev);
5965
5966                 if (memsize != 0xffffffff)
5967                         break;
5968                 udelay(1);
5969         }
5970         if (memsize == 0xffffffff) {
5971                 r = -ETIME;
5972                 goto out;
5973         }
5974
5975         reset_context.method = AMD_RESET_METHOD_NONE;
5976         reset_context.reset_req_dev = adev;
5977         set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
5978         set_bit(AMDGPU_SKIP_HW_RESET, &reset_context.flags);
5979
5980         adev->no_hw_access = true;
5981         r = amdgpu_device_pre_asic_reset(adev, &reset_context);
5982         adev->no_hw_access = false;
5983         if (r)
5984                 goto out;
5985
5986         r = amdgpu_do_asic_reset(&device_list, &reset_context);
5987
5988 out:
5989         if (!r) {
5990                 if (amdgpu_device_cache_pci_state(adev->pdev))
5991                         pci_restore_state(adev->pdev);
5992
5993                 DRM_INFO("PCIe error recovery succeeded\n");
5994         } else {
5995                 DRM_ERROR("PCIe error recovery failed, err:%d", r);
5996                 amdgpu_device_unset_mp1_state(adev);
5997                 amdgpu_device_unlock_reset_domain(adev->reset_domain);
5998         }
5999
6000         return r ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_RECOVERED;
6001 }
6002
6003 /**
6004  * amdgpu_pci_resume() - resume normal ops after PCI reset
6005  * @pdev: pointer to PCI device
6006  *
6007  * Called when the error recovery driver tells us that its
6008  * OK to resume normal operation.
6009  */
6010 void amdgpu_pci_resume(struct pci_dev *pdev)
6011 {
6012         struct drm_device *dev = pci_get_drvdata(pdev);
6013         struct amdgpu_device *adev = drm_to_adev(dev);
6014         int i;
6015
6016
6017         DRM_INFO("PCI error: resume callback!!\n");
6018
6019         /* Only continue execution for the case of pci_channel_io_frozen */
6020         if (adev->pci_channel_state != pci_channel_io_frozen)
6021                 return;
6022
6023         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
6024                 struct amdgpu_ring *ring = adev->rings[i];
6025
6026                 if (!ring || !ring->sched.thread)
6027                         continue;
6028
6029                 drm_sched_start(&ring->sched, true);
6030         }
6031
6032         amdgpu_device_unset_mp1_state(adev);
6033         amdgpu_device_unlock_reset_domain(adev->reset_domain);
6034 }
6035
6036 bool amdgpu_device_cache_pci_state(struct pci_dev *pdev)
6037 {
6038         struct drm_device *dev = pci_get_drvdata(pdev);
6039         struct amdgpu_device *adev = drm_to_adev(dev);
6040         int r;
6041
6042         r = pci_save_state(pdev);
6043         if (!r) {
6044                 kfree(adev->pci_state);
6045
6046                 adev->pci_state = pci_store_saved_state(pdev);
6047
6048                 if (!adev->pci_state) {
6049                         DRM_ERROR("Failed to store PCI saved state");
6050                         return false;
6051                 }
6052         } else {
6053                 DRM_WARN("Failed to save PCI state, err:%d\n", r);
6054                 return false;
6055         }
6056
6057         return true;
6058 }
6059
6060 bool amdgpu_device_load_pci_state(struct pci_dev *pdev)
6061 {
6062         struct drm_device *dev = pci_get_drvdata(pdev);
6063         struct amdgpu_device *adev = drm_to_adev(dev);
6064         int r;
6065
6066         if (!adev->pci_state)
6067                 return false;
6068
6069         r = pci_load_saved_state(pdev, adev->pci_state);
6070
6071         if (!r) {
6072                 pci_restore_state(pdev);
6073         } else {
6074                 DRM_WARN("Failed to load PCI state, err:%d\n", r);
6075                 return false;
6076         }
6077
6078         return true;
6079 }
6080
6081 void amdgpu_device_flush_hdp(struct amdgpu_device *adev,
6082                 struct amdgpu_ring *ring)
6083 {
6084 #ifdef CONFIG_X86_64
6085         if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev))
6086                 return;
6087 #endif
6088         if (adev->gmc.xgmi.connected_to_cpu)
6089                 return;
6090
6091         if (ring && ring->funcs->emit_hdp_flush)
6092                 amdgpu_ring_emit_hdp_flush(ring);
6093         else
6094                 amdgpu_asic_flush_hdp(adev, ring);
6095 }
6096
6097 void amdgpu_device_invalidate_hdp(struct amdgpu_device *adev,
6098                 struct amdgpu_ring *ring)
6099 {
6100 #ifdef CONFIG_X86_64
6101         if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev))
6102                 return;
6103 #endif
6104         if (adev->gmc.xgmi.connected_to_cpu)
6105                 return;
6106
6107         amdgpu_asic_invalidate_hdp(adev, ring);
6108 }
6109
6110 int amdgpu_in_reset(struct amdgpu_device *adev)
6111 {
6112         return atomic_read(&adev->reset_domain->in_gpu_reset);
6113 }
6114
6115 /**
6116  * amdgpu_device_halt() - bring hardware to some kind of halt state
6117  *
6118  * @adev: amdgpu_device pointer
6119  *
6120  * Bring hardware to some kind of halt state so that no one can touch it
6121  * any more. It will help to maintain error context when error occurred.
6122  * Compare to a simple hang, the system will keep stable at least for SSH
6123  * access. Then it should be trivial to inspect the hardware state and
6124  * see what's going on. Implemented as following:
6125  *
6126  * 1. drm_dev_unplug() makes device inaccessible to user space(IOCTLs, etc),
6127  *    clears all CPU mappings to device, disallows remappings through page faults
6128  * 2. amdgpu_irq_disable_all() disables all interrupts
6129  * 3. amdgpu_fence_driver_hw_fini() signals all HW fences
6130  * 4. set adev->no_hw_access to avoid potential crashes after setp 5
6131  * 5. amdgpu_device_unmap_mmio() clears all MMIO mappings
6132  * 6. pci_disable_device() and pci_wait_for_pending_transaction()
6133  *    flush any in flight DMA operations
6134  */
6135 void amdgpu_device_halt(struct amdgpu_device *adev)
6136 {
6137         struct pci_dev *pdev = adev->pdev;
6138         struct drm_device *ddev = adev_to_drm(adev);
6139
6140         amdgpu_xcp_dev_unplug(adev);
6141         drm_dev_unplug(ddev);
6142
6143         amdgpu_irq_disable_all(adev);
6144
6145         amdgpu_fence_driver_hw_fini(adev);
6146
6147         adev->no_hw_access = true;
6148
6149         amdgpu_device_unmap_mmio(adev);
6150
6151         pci_disable_device(pdev);
6152         pci_wait_for_pending_transaction(pdev);
6153 }
6154
6155 u32 amdgpu_device_pcie_port_rreg(struct amdgpu_device *adev,
6156                                 u32 reg)
6157 {
6158         unsigned long flags, address, data;
6159         u32 r;
6160
6161         address = adev->nbio.funcs->get_pcie_port_index_offset(adev);
6162         data = adev->nbio.funcs->get_pcie_port_data_offset(adev);
6163
6164         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
6165         WREG32(address, reg * 4);
6166         (void)RREG32(address);
6167         r = RREG32(data);
6168         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
6169         return r;
6170 }
6171
6172 void amdgpu_device_pcie_port_wreg(struct amdgpu_device *adev,
6173                                 u32 reg, u32 v)
6174 {
6175         unsigned long flags, address, data;
6176
6177         address = adev->nbio.funcs->get_pcie_port_index_offset(adev);
6178         data = adev->nbio.funcs->get_pcie_port_data_offset(adev);
6179
6180         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
6181         WREG32(address, reg * 4);
6182         (void)RREG32(address);
6183         WREG32(data, v);
6184         (void)RREG32(data);
6185         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
6186 }
6187
6188 /**
6189  * amdgpu_device_switch_gang - switch to a new gang
6190  * @adev: amdgpu_device pointer
6191  * @gang: the gang to switch to
6192  *
6193  * Try to switch to a new gang.
6194  * Returns: NULL if we switched to the new gang or a reference to the current
6195  * gang leader.
6196  */
6197 struct dma_fence *amdgpu_device_switch_gang(struct amdgpu_device *adev,
6198                                             struct dma_fence *gang)
6199 {
6200         struct dma_fence *old = NULL;
6201
6202         do {
6203                 dma_fence_put(old);
6204                 rcu_read_lock();
6205                 old = dma_fence_get_rcu_safe(&adev->gang_submit);
6206                 rcu_read_unlock();
6207
6208                 if (old == gang)
6209                         break;
6210
6211                 if (!dma_fence_is_signaled(old))
6212                         return old;
6213
6214         } while (cmpxchg((struct dma_fence __force **)&adev->gang_submit,
6215                          old, gang) != old);
6216
6217         dma_fence_put(old);
6218         return NULL;
6219 }
6220
6221 bool amdgpu_device_has_display_hardware(struct amdgpu_device *adev)
6222 {
6223         switch (adev->asic_type) {
6224 #ifdef CONFIG_DRM_AMDGPU_SI
6225         case CHIP_HAINAN:
6226 #endif
6227         case CHIP_TOPAZ:
6228                 /* chips with no display hardware */
6229                 return false;
6230 #ifdef CONFIG_DRM_AMDGPU_SI
6231         case CHIP_TAHITI:
6232         case CHIP_PITCAIRN:
6233         case CHIP_VERDE:
6234         case CHIP_OLAND:
6235 #endif
6236 #ifdef CONFIG_DRM_AMDGPU_CIK
6237         case CHIP_BONAIRE:
6238         case CHIP_HAWAII:
6239         case CHIP_KAVERI:
6240         case CHIP_KABINI:
6241         case CHIP_MULLINS:
6242 #endif
6243         case CHIP_TONGA:
6244         case CHIP_FIJI:
6245         case CHIP_POLARIS10:
6246         case CHIP_POLARIS11:
6247         case CHIP_POLARIS12:
6248         case CHIP_VEGAM:
6249         case CHIP_CARRIZO:
6250         case CHIP_STONEY:
6251                 /* chips with display hardware */
6252                 return true;
6253         default:
6254                 /* IP discovery */
6255                 if (!amdgpu_ip_version(adev, DCE_HWIP, 0) ||
6256                     (adev->harvest_ip_mask & AMD_HARVEST_IP_DMU_MASK))
6257                         return false;
6258                 return true;
6259         }
6260 }
6261
6262 uint32_t amdgpu_device_wait_on_rreg(struct amdgpu_device *adev,
6263                 uint32_t inst, uint32_t reg_addr, char reg_name[],
6264                 uint32_t expected_value, uint32_t mask)
6265 {
6266         uint32_t ret = 0;
6267         uint32_t old_ = 0;
6268         uint32_t tmp_ = RREG32(reg_addr);
6269         uint32_t loop = adev->usec_timeout;
6270
6271         while ((tmp_ & (mask)) != (expected_value)) {
6272                 if (old_ != tmp_) {
6273                         loop = adev->usec_timeout;
6274                         old_ = tmp_;
6275                 } else
6276                         udelay(1);
6277                 tmp_ = RREG32(reg_addr);
6278                 loop--;
6279                 if (!loop) {
6280                         DRM_WARN("Register(%d) [%s] failed to reach value 0x%08x != 0x%08xn",
6281                                   inst, reg_name, (uint32_t)expected_value,
6282                                   (uint32_t)(tmp_ & (mask)));
6283                         ret = -ETIMEDOUT;
6284                         break;
6285                 }
6286         }
6287         return ret;
6288 }