drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c

   1 /*
   2  * Copyright 2008 Advanced Micro Devices, Inc.
   3  * Copyright 2008 Red Hat Inc.
   4  * Copyright 2009 Jerome Glisse.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22  * OTHER DEALINGS IN THE SOFTWARE.
  23  *
  24  */
  25
  26 #include <linux/kthread.h>
  27 #include <linux/pci.h>
  28 #include <linux/uaccess.h>
  29 #include <linux/pm_runtime.h>
  30 #include <linux/poll.h>
  31 #include <drm/drm_debugfs.h>
  32
  33 #include "amdgpu.h"
  34 #include "amdgpu_pm.h"
  35 #include "amdgpu_dm_debugfs.h"
  36 #include "amdgpu_ras.h"
  37 #include "amdgpu_rap.h"
  38
  39 /**
  40  * amdgpu_debugfs_add_files - Add simple debugfs entries
  41  *
  42  * @adev:  Device to attach debugfs entries to
  43  * @files:  Array of function callbacks that respond to reads
  44  * @nfiles: Number of callbacks to register
  45  *
  46  */
  47 int amdgpu_debugfs_add_files(struct amdgpu_device *adev,
  48                              const struct drm_info_list *files,
  49                              unsigned nfiles)
  50 {
  51         unsigned i;
  52
  53         for (i = 0; i < adev->debugfs_count; i++) {
  54                 if (adev->debugfs[i].files == files) {
  55                         /* Already registered */
  56                         return 0;
  57                 }
  58         }
  59
  60         i = adev->debugfs_count + 1;
  61         if (i > AMDGPU_DEBUGFS_MAX_COMPONENTS) {
  62                 DRM_ERROR("Reached maximum number of debugfs components.\n");
  63                 DRM_ERROR("Report so we increase "
  64                           "AMDGPU_DEBUGFS_MAX_COMPONENTS.\n");
  65                 return -EINVAL;
  66         }
  67         adev->debugfs[adev->debugfs_count].files = files;
  68         adev->debugfs[adev->debugfs_count].num_files = nfiles;
  69         adev->debugfs_count = i;
  70 #if defined(CONFIG_DEBUG_FS)
  71         drm_debugfs_create_files(files, nfiles,
  72                                  adev->ddev->primary->debugfs_root,
  73                                  adev->ddev->primary);
  74 #endif
  75         return 0;
  76 }
  77
  78 int amdgpu_debugfs_wait_dump(struct amdgpu_device *adev)
  79 {
  80 #if defined(CONFIG_DEBUG_FS)
  81         unsigned long timeout = 600 * HZ;
  82         int ret;
  83
  84         wake_up_interruptible(&adev->autodump.gpu_hang);
  85
  86         ret = wait_for_completion_interruptible_timeout(&adev->autodump.dumping, timeout);
  87         if (ret == 0) {
  88                 pr_err("autodump: timeout, move on to gpu recovery\n");
  89                 return -ETIMEDOUT;
  90         }
  91 #endif
  92         return 0;
  93 }
  94
  95 #if defined(CONFIG_DEBUG_FS)
  96
  97 static int amdgpu_debugfs_autodump_open(struct inode *inode, struct file *file)
  98 {
  99         struct amdgpu_device *adev = inode->i_private;
 100         int ret;
 101
 102         file->private_data = adev;
 103
 104         mutex_lock(&adev->lock_reset);
 105         if (adev->autodump.dumping.done) {
 106                 reinit_completion(&adev->autodump.dumping);
 107                 ret = 0;
 108         } else {
 109                 ret = -EBUSY;
 110         }
 111         mutex_unlock(&adev->lock_reset);
 112
 113         return ret;
 114 }
 115
 116 static int amdgpu_debugfs_autodump_release(struct inode *inode, struct file *file)
 117 {
 118         struct amdgpu_device *adev = file->private_data;
 119
 120         complete_all(&adev->autodump.dumping);
 121         return 0;
 122 }
 123
 124 static unsigned int amdgpu_debugfs_autodump_poll(struct file *file, struct poll_table_struct *poll_table)
 125 {
 126         struct amdgpu_device *adev = file->private_data;
 127
 128         poll_wait(file, &adev->autodump.gpu_hang, poll_table);
 129
 130         if (amdgpu_in_reset(adev))
 131                 return POLLIN | POLLRDNORM | POLLWRNORM;
 132
 133         return 0;
 134 }
 135
 136 static const struct file_operations autodump_debug_fops = {
 137         .owner = THIS_MODULE,
 138         .open = amdgpu_debugfs_autodump_open,
 139         .poll = amdgpu_debugfs_autodump_poll,
 140         .release = amdgpu_debugfs_autodump_release,
 141 };
 142
 143 static void amdgpu_debugfs_autodump_init(struct amdgpu_device *adev)
 144 {
 145         init_completion(&adev->autodump.dumping);
 146         complete_all(&adev->autodump.dumping);
 147         init_waitqueue_head(&adev->autodump.gpu_hang);
 148
 149         debugfs_create_file("amdgpu_autodump", 0600,
 150                 adev->ddev->primary->debugfs_root,
 151                 adev, &autodump_debug_fops);
 152 }
 153
 154 /**
 155  * amdgpu_debugfs_process_reg_op - Handle MMIO register reads/writes
 156  *
 157  * @read: True if reading
 158  * @f: open file handle
 159  * @buf: User buffer to write/read to
 160  * @size: Number of bytes to write/read
 161  * @pos:  Offset to seek to
 162  *
 163  * This debugfs entry has special meaning on the offset being sought.
 164  * Various bits have different meanings:
 165  *
 166  * Bit 62:  Indicates a GRBM bank switch is needed
 167  * Bit 61:  Indicates a SRBM bank switch is needed (implies bit 62 is
 168  *                      zero)
 169  * Bits 24..33: The SE or ME selector if needed
 170  * Bits 34..43: The SH (or SA) or PIPE selector if needed
 171  * Bits 44..53: The INSTANCE (or CU/WGP) or QUEUE selector if needed
 172  *
 173  * Bit 23:  Indicates that the PM power gating lock should be held
 174  *                      This is necessary to read registers that might be
 175  *                      unreliable during a power gating transistion.
 176  *
 177  * The lower bits are the BYTE offset of the register to read.  This
 178  * allows reading multiple registers in a single call and having
 179  * the returned size reflect that.
 180  */
 181 static int  amdgpu_debugfs_process_reg_op(bool read, struct file *f,
 182                 char __user *buf, size_t size, loff_t *pos)
 183 {
 184         struct amdgpu_device *adev = file_inode(f)->i_private;
 185         ssize_t result = 0;
 186         int r;
 187         bool pm_pg_lock, use_bank, use_ring;
 188         unsigned instance_bank, sh_bank, se_bank, me, pipe, queue, vmid;
 189
 190         pm_pg_lock = use_bank = use_ring = false;
 191         instance_bank = sh_bank = se_bank = me = pipe = queue = vmid = 0;
 192
 193         if (size & 0x3 || *pos & 0x3 ||
 194                         ((*pos & (1ULL << 62)) && (*pos & (1ULL << 61))))
 195                 return -EINVAL;
 196
 197         /* are we reading registers for which a PG lock is necessary? */
 198         pm_pg_lock = (*pos >> 23) & 1;
 199
 200         if (*pos & (1ULL << 62)) {
 201                 se_bank = (*pos & GENMASK_ULL(33, 24)) >> 24;
 202                 sh_bank = (*pos & GENMASK_ULL(43, 34)) >> 34;
 203                 instance_bank = (*pos & GENMASK_ULL(53, 44)) >> 44;
 204
 205                 if (se_bank == 0x3FF)
 206                         se_bank = 0xFFFFFFFF;
 207                 if (sh_bank == 0x3FF)
 208                         sh_bank = 0xFFFFFFFF;
 209                 if (instance_bank == 0x3FF)
 210                         instance_bank = 0xFFFFFFFF;
 211                 use_bank = true;
 212         } else if (*pos & (1ULL << 61)) {
 213
 214                 me = (*pos & GENMASK_ULL(33, 24)) >> 24;
 215                 pipe = (*pos & GENMASK_ULL(43, 34)) >> 34;
 216                 queue = (*pos & GENMASK_ULL(53, 44)) >> 44;
 217                 vmid = (*pos & GENMASK_ULL(58, 54)) >> 54;
 218
 219                 use_ring = true;
 220         } else {
 221                 use_bank = use_ring = false;
 222         }
 223
 224         *pos &= (1UL << 22) - 1;
 225
 226         r = pm_runtime_get_sync(adev->ddev->dev);
 227         if (r < 0) {
 228                 pm_runtime_put_autosuspend(adev->ddev->dev);
 229                 return r;
 230         }
 231
 232         r = amdgpu_virt_enable_access_debugfs(adev);
 233         if (r < 0) {
 234                 pm_runtime_put_autosuspend(adev->ddev->dev);
 235                 return r;
 236         }
 237
 238         if (use_bank) {
 239                 if ((sh_bank != 0xFFFFFFFF && sh_bank >= adev->gfx.config.max_sh_per_se) ||
 240                     (se_bank != 0xFFFFFFFF && se_bank >= adev->gfx.config.max_shader_engines)) {
 241                         pm_runtime_mark_last_busy(adev->ddev->dev);
 242                         pm_runtime_put_autosuspend(adev->ddev->dev);
 243                         amdgpu_virt_disable_access_debugfs(adev);
 244                         return -EINVAL;
 245                 }
 246                 mutex_lock(&adev->grbm_idx_mutex);
 247                 amdgpu_gfx_select_se_sh(adev, se_bank,
 248                                         sh_bank, instance_bank);
 249         } else if (use_ring) {
 250                 mutex_lock(&adev->srbm_mutex);
 251                 amdgpu_gfx_select_me_pipe_q(adev, me, pipe, queue, vmid);
 252         }
 253
 254         if (pm_pg_lock)
 255                 mutex_lock(&adev->pm.mutex);
 256
 257         while (size) {
 258                 uint32_t value;
 259
 260                 if (read) {
 261                         value = RREG32(*pos >> 2);
 262                         r = put_user(value, (uint32_t *)buf);
 263                 } else {
 264                         r = get_user(value, (uint32_t *)buf);
 265                         if (!r)
 266                                 amdgpu_mm_wreg_mmio_rlc(adev, *pos >> 2, value, 0);
 267                 }
 268                 if (r) {
 269                         result = r;
 270                         goto end;
 271                 }
 272
 273                 result += 4;
 274                 buf += 4;
 275                 *pos += 4;
 276                 size -= 4;
 277         }
 278
 279 end:
 280         if (use_bank) {
 281                 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
 282                 mutex_unlock(&adev->grbm_idx_mutex);
 283         } else if (use_ring) {
 284                 amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0);
 285                 mutex_unlock(&adev->srbm_mutex);
 286         }
 287
 288         if (pm_pg_lock)
 289                 mutex_unlock(&adev->pm.mutex);
 290
 291         pm_runtime_mark_last_busy(adev->ddev->dev);
 292         pm_runtime_put_autosuspend(adev->ddev->dev);
 293
 294         amdgpu_virt_disable_access_debugfs(adev);
 295         return result;
 296 }
 297
 298 /**
 299  * amdgpu_debugfs_regs_read - Callback for reading MMIO registers
 300  */
 301 static ssize_t amdgpu_debugfs_regs_read(struct file *f, char __user *buf,
 302                                         size_t size, loff_t *pos)
 303 {
 304         return amdgpu_debugfs_process_reg_op(true, f, buf, size, pos);
 305 }
 306
 307 /**
 308  * amdgpu_debugfs_regs_write - Callback for writing MMIO registers
 309  */
 310 static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,
 311                                          size_t size, loff_t *pos)
 312 {
 313         return amdgpu_debugfs_process_reg_op(false, f, (char __user *)buf, size, pos);
 314 }
 315
 316
 317 /**
 318  * amdgpu_debugfs_regs_pcie_read - Read from a PCIE register
 319  *
 320  * @f: open file handle
 321  * @buf: User buffer to store read data in
 322  * @size: Number of bytes to read
 323  * @pos:  Offset to seek to
 324  *
 325  * The lower bits are the BYTE offset of the register to read.  This
 326  * allows reading multiple registers in a single call and having
 327  * the returned size reflect that.
 328  */
 329 static ssize_t amdgpu_debugfs_regs_pcie_read(struct file *f, char __user *buf,
 330                                         size_t size, loff_t *pos)
 331 {
 332         struct amdgpu_device *adev = file_inode(f)->i_private;
 333         ssize_t result = 0;
 334         int r;
 335
 336         if (size & 0x3 || *pos & 0x3)
 337                 return -EINVAL;
 338
 339         r = pm_runtime_get_sync(adev->ddev->dev);
 340         if (r < 0) {
 341                 pm_runtime_put_autosuspend(adev->ddev->dev);
 342                 return r;
 343         }
 344
 345         r = amdgpu_virt_enable_access_debugfs(adev);
 346         if (r < 0) {
 347                 pm_runtime_put_autosuspend(adev->ddev->dev);
 348                 return r;
 349         }
 350
 351         while (size) {
 352                 uint32_t value;
 353
 354                 value = RREG32_PCIE(*pos >> 2);
 355                 r = put_user(value, (uint32_t *)buf);
 356                 if (r) {
 357                         pm_runtime_mark_last_busy(adev->ddev->dev);
 358                         pm_runtime_put_autosuspend(adev->ddev->dev);
 359                         amdgpu_virt_disable_access_debugfs(adev);
 360                         return r;
 361                 }
 362
 363                 result += 4;
 364                 buf += 4;
 365                 *pos += 4;
 366                 size -= 4;
 367         }
 368
 369         pm_runtime_mark_last_busy(adev->ddev->dev);
 370         pm_runtime_put_autosuspend(adev->ddev->dev);
 371
 372         amdgpu_virt_disable_access_debugfs(adev);
 373         return result;
 374 }
 375
 376 /**
 377  * amdgpu_debugfs_regs_pcie_write - Write to a PCIE register
 378  *
 379  * @f: open file handle
 380  * @buf: User buffer to write data from
 381  * @size: Number of bytes to write
 382  * @pos:  Offset to seek to
 383  *
 384  * The lower bits are the BYTE offset of the register to write.  This
 385  * allows writing multiple registers in a single call and having
 386  * the returned size reflect that.
 387  */
 388 static ssize_t amdgpu_debugfs_regs_pcie_write(struct file *f, const char __user *buf,
 389                                          size_t size, loff_t *pos)
 390 {
 391         struct amdgpu_device *adev = file_inode(f)->i_private;
 392         ssize_t result = 0;
 393         int r;
 394
 395         if (size & 0x3 || *pos & 0x3)
 396                 return -EINVAL;
 397
 398         r = pm_runtime_get_sync(adev->ddev->dev);
 399         if (r < 0) {
 400                 pm_runtime_put_autosuspend(adev->ddev->dev);
 401                 return r;
 402         }
 403
 404         r = amdgpu_virt_enable_access_debugfs(adev);
 405         if (r < 0) {
 406                 pm_runtime_put_autosuspend(adev->ddev->dev);
 407                 return r;
 408         }
 409
 410         while (size) {
 411                 uint32_t value;
 412
 413                 r = get_user(value, (uint32_t *)buf);
 414                 if (r) {
 415                         pm_runtime_mark_last_busy(adev->ddev->dev);
 416                         pm_runtime_put_autosuspend(adev->ddev->dev);
 417                         amdgpu_virt_disable_access_debugfs(adev);
 418                         return r;
 419                 }
 420
 421                 WREG32_PCIE(*pos >> 2, value);
 422
 423                 result += 4;
 424                 buf += 4;
 425                 *pos += 4;
 426                 size -= 4;
 427         }
 428
 429         pm_runtime_mark_last_busy(adev->ddev->dev);
 430         pm_runtime_put_autosuspend(adev->ddev->dev);
 431
 432         amdgpu_virt_disable_access_debugfs(adev);
 433         return result;
 434 }
 435
 436 /**
 437  * amdgpu_debugfs_regs_didt_read - Read from a DIDT register
 438  *
 439  * @f: open file handle
 440  * @buf: User buffer to store read data in
 441  * @size: Number of bytes to read
 442  * @pos:  Offset to seek to
 443  *
 444  * The lower bits are the BYTE offset of the register to read.  This
 445  * allows reading multiple registers in a single call and having
 446  * the returned size reflect that.
 447  */
 448 static ssize_t amdgpu_debugfs_regs_didt_read(struct file *f, char __user *buf,
 449                                         size_t size, loff_t *pos)
 450 {
 451         struct amdgpu_device *adev = file_inode(f)->i_private;
 452         ssize_t result = 0;
 453         int r;
 454
 455         if (size & 0x3 || *pos & 0x3)
 456                 return -EINVAL;
 457
 458         r = pm_runtime_get_sync(adev->ddev->dev);
 459         if (r < 0) {
 460                 pm_runtime_put_autosuspend(adev->ddev->dev);
 461                 return r;
 462         }
 463
 464         r = amdgpu_virt_enable_access_debugfs(adev);
 465         if (r < 0) {
 466                 pm_runtime_put_autosuspend(adev->ddev->dev);
 467                 return r;
 468         }
 469
 470         while (size) {
 471                 uint32_t value;
 472
 473                 value = RREG32_DIDT(*pos >> 2);
 474                 r = put_user(value, (uint32_t *)buf);
 475                 if (r) {
 476                         pm_runtime_mark_last_busy(adev->ddev->dev);
 477                         pm_runtime_put_autosuspend(adev->ddev->dev);
 478                         amdgpu_virt_disable_access_debugfs(adev);
 479                         return r;
 480                 }
 481
 482                 result += 4;
 483                 buf += 4;
 484                 *pos += 4;
 485                 size -= 4;
 486         }
 487
 488         pm_runtime_mark_last_busy(adev->ddev->dev);
 489         pm_runtime_put_autosuspend(adev->ddev->dev);
 490
 491         amdgpu_virt_disable_access_debugfs(adev);
 492         return result;
 493 }
 494
 495 /**
 496  * amdgpu_debugfs_regs_didt_write - Write to a DIDT register
 497  *
 498  * @f: open file handle
 499  * @buf: User buffer to write data from
 500  * @size: Number of bytes to write
 501  * @pos:  Offset to seek to
 502  *
 503  * The lower bits are the BYTE offset of the register to write.  This
 504  * allows writing multiple registers in a single call and having
 505  * the returned size reflect that.
 506  */
 507 static ssize_t amdgpu_debugfs_regs_didt_write(struct file *f, const char __user *buf,
 508                                          size_t size, loff_t *pos)
 509 {
 510         struct amdgpu_device *adev = file_inode(f)->i_private;
 511         ssize_t result = 0;
 512         int r;
 513
 514         if (size & 0x3 || *pos & 0x3)
 515                 return -EINVAL;
 516
 517         r = pm_runtime_get_sync(adev->ddev->dev);
 518         if (r < 0) {
 519                 pm_runtime_put_autosuspend(adev->ddev->dev);
 520                 return r;
 521         }
 522
 523         r = amdgpu_virt_enable_access_debugfs(adev);
 524         if (r < 0) {
 525                 pm_runtime_put_autosuspend(adev->ddev->dev);
 526                 return r;
 527         }
 528
 529         while (size) {
 530                 uint32_t value;
 531
 532                 r = get_user(value, (uint32_t *)buf);
 533                 if (r) {
 534                         pm_runtime_mark_last_busy(adev->ddev->dev);
 535                         pm_runtime_put_autosuspend(adev->ddev->dev);
 536                         amdgpu_virt_disable_access_debugfs(adev);
 537                         return r;
 538                 }
 539
 540                 WREG32_DIDT(*pos >> 2, value);
 541
 542                 result += 4;
 543                 buf += 4;
 544                 *pos += 4;
 545                 size -= 4;
 546         }
 547
 548         pm_runtime_mark_last_busy(adev->ddev->dev);
 549         pm_runtime_put_autosuspend(adev->ddev->dev);
 550
 551         amdgpu_virt_disable_access_debugfs(adev);
 552         return result;
 553 }
 554
 555 /**
 556  * amdgpu_debugfs_regs_smc_read - Read from a SMC register
 557  *
 558  * @f: open file handle
 559  * @buf: User buffer to store read data in
 560  * @size: Number of bytes to read
 561  * @pos:  Offset to seek to
 562  *
 563  * The lower bits are the BYTE offset of the register to read.  This
 564  * allows reading multiple registers in a single call and having
 565  * the returned size reflect that.
 566  */
 567 static ssize_t amdgpu_debugfs_regs_smc_read(struct file *f, char __user *buf,
 568                                         size_t size, loff_t *pos)
 569 {
 570         struct amdgpu_device *adev = file_inode(f)->i_private;
 571         ssize_t result = 0;
 572         int r;
 573
 574         if (size & 0x3 || *pos & 0x3)
 575                 return -EINVAL;
 576
 577         r = pm_runtime_get_sync(adev->ddev->dev);
 578         if (r < 0) {
 579                 pm_runtime_put_autosuspend(adev->ddev->dev);
 580                 return r;
 581         }
 582
 583         r = amdgpu_virt_enable_access_debugfs(adev);
 584         if (r < 0) {
 585                 pm_runtime_put_autosuspend(adev->ddev->dev);
 586                 return r;
 587         }
 588
 589         while (size) {
 590                 uint32_t value;
 591
 592                 value = RREG32_SMC(*pos);
 593                 r = put_user(value, (uint32_t *)buf);
 594                 if (r) {
 595                         pm_runtime_mark_last_busy(adev->ddev->dev);
 596                         pm_runtime_put_autosuspend(adev->ddev->dev);
 597                         amdgpu_virt_disable_access_debugfs(adev);
 598                         return r;
 599                 }
 600
 601                 result += 4;
 602                 buf += 4;
 603                 *pos += 4;
 604                 size -= 4;
 605         }
 606
 607         pm_runtime_mark_last_busy(adev->ddev->dev);
 608         pm_runtime_put_autosuspend(adev->ddev->dev);
 609
 610         amdgpu_virt_disable_access_debugfs(adev);
 611         return result;
 612 }
 613
 614 /**
 615  * amdgpu_debugfs_regs_smc_write - Write to a SMC register
 616  *
 617  * @f: open file handle
 618  * @buf: User buffer to write data from
 619  * @size: Number of bytes to write
 620  * @pos:  Offset to seek to
 621  *
 622  * The lower bits are the BYTE offset of the register to write.  This
 623  * allows writing multiple registers in a single call and having
 624  * the returned size reflect that.
 625  */
 626 static ssize_t amdgpu_debugfs_regs_smc_write(struct file *f, const char __user *buf,
 627                                          size_t size, loff_t *pos)
 628 {
 629         struct amdgpu_device *adev = file_inode(f)->i_private;
 630         ssize_t result = 0;
 631         int r;
 632
 633         if (size & 0x3 || *pos & 0x3)
 634                 return -EINVAL;
 635
 636         r = pm_runtime_get_sync(adev->ddev->dev);
 637         if (r < 0) {
 638                 pm_runtime_put_autosuspend(adev->ddev->dev);
 639                 return r;
 640         }
 641
 642         r = amdgpu_virt_enable_access_debugfs(adev);
 643         if (r < 0) {
 644                 pm_runtime_put_autosuspend(adev->ddev->dev);
 645                 return r;
 646         }
 647
 648         while (size) {
 649                 uint32_t value;
 650
 651                 r = get_user(value, (uint32_t *)buf);
 652                 if (r) {
 653                         pm_runtime_mark_last_busy(adev->ddev->dev);
 654                         pm_runtime_put_autosuspend(adev->ddev->dev);
 655                         amdgpu_virt_disable_access_debugfs(adev);
 656                         return r;
 657                 }
 658
 659                 WREG32_SMC(*pos, value);
 660
 661                 result += 4;
 662                 buf += 4;
 663                 *pos += 4;
 664                 size -= 4;
 665         }
 666
 667         pm_runtime_mark_last_busy(adev->ddev->dev);
 668         pm_runtime_put_autosuspend(adev->ddev->dev);
 669
 670         amdgpu_virt_disable_access_debugfs(adev);
 671         return result;
 672 }
 673
 674 /**
 675  * amdgpu_debugfs_gca_config_read - Read from gfx config data
 676  *
 677  * @f: open file handle
 678  * @buf: User buffer to store read data in
 679  * @size: Number of bytes to read
 680  * @pos:  Offset to seek to
 681  *
 682  * This file is used to access configuration data in a somewhat
 683  * stable fashion.  The format is a series of DWORDs with the first
 684  * indicating which revision it is.  New content is appended to the
 685  * end so that older software can still read the data.
 686  */
 687
 688 static ssize_t amdgpu_debugfs_gca_config_read(struct file *f, char __user *buf,
 689                                         size_t size, loff_t *pos)
 690 {
 691         struct amdgpu_device *adev = file_inode(f)->i_private;
 692         ssize_t result = 0;
 693         int r;
 694         uint32_t *config, no_regs = 0;
 695
 696         if (size & 0x3 || *pos & 0x3)
 697                 return -EINVAL;
 698
 699         config = kmalloc_array(256, sizeof(*config), GFP_KERNEL);
 700         if (!config)
 701                 return -ENOMEM;
 702
 703         /* version, increment each time something is added */
 704         config[no_regs++] = 3;
 705         config[no_regs++] = adev->gfx.config.max_shader_engines;
 706         config[no_regs++] = adev->gfx.config.max_tile_pipes;
 707         config[no_regs++] = adev->gfx.config.max_cu_per_sh;
 708         config[no_regs++] = adev->gfx.config.max_sh_per_se;
 709         config[no_regs++] = adev->gfx.config.max_backends_per_se;
 710         config[no_regs++] = adev->gfx.config.max_texture_channel_caches;
 711         config[no_regs++] = adev->gfx.config.max_gprs;
 712         config[no_regs++] = adev->gfx.config.max_gs_threads;
 713         config[no_regs++] = adev->gfx.config.max_hw_contexts;
 714         config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_frontend;
 715         config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_backend;
 716         config[no_regs++] = adev->gfx.config.sc_hiz_tile_fifo_size;
 717         config[no_regs++] = adev->gfx.config.sc_earlyz_tile_fifo_size;
 718         config[no_regs++] = adev->gfx.config.num_tile_pipes;
 719         config[no_regs++] = adev->gfx.config.backend_enable_mask;
 720         config[no_regs++] = adev->gfx.config.mem_max_burst_length_bytes;
 721         config[no_regs++] = adev->gfx.config.mem_row_size_in_kb;
 722         config[no_regs++] = adev->gfx.config.shader_engine_tile_size;
 723         config[no_regs++] = adev->gfx.config.num_gpus;
 724         config[no_regs++] = adev->gfx.config.multi_gpu_tile_size;
 725         config[no_regs++] = adev->gfx.config.mc_arb_ramcfg;
 726         config[no_regs++] = adev->gfx.config.gb_addr_config;
 727         config[no_regs++] = adev->gfx.config.num_rbs;
 728
 729         /* rev==1 */
 730         config[no_regs++] = adev->rev_id;
 731         config[no_regs++] = adev->pg_flags;
 732         config[no_regs++] = adev->cg_flags;
 733
 734         /* rev==2 */
 735         config[no_regs++] = adev->family;
 736         config[no_regs++] = adev->external_rev_id;
 737
 738         /* rev==3 */
 739         config[no_regs++] = adev->pdev->device;
 740         config[no_regs++] = adev->pdev->revision;
 741         config[no_regs++] = adev->pdev->subsystem_device;
 742         config[no_regs++] = adev->pdev->subsystem_vendor;
 743
 744         while (size && (*pos < no_regs * 4)) {
 745                 uint32_t value;
 746
 747                 value = config[*pos >> 2];
 748                 r = put_user(value, (uint32_t *)buf);
 749                 if (r) {
 750                         kfree(config);
 751                         return r;
 752                 }
 753
 754                 result += 4;
 755                 buf += 4;
 756                 *pos += 4;
 757                 size -= 4;
 758         }
 759
 760         kfree(config);
 761         return result;
 762 }
 763
 764 /**
 765  * amdgpu_debugfs_sensor_read - Read from the powerplay sensors
 766  *
 767  * @f: open file handle
 768  * @buf: User buffer to store read data in
 769  * @size: Number of bytes to read
 770  * @pos:  Offset to seek to
 771  *
 772  * The offset is treated as the BYTE address of one of the sensors
 773  * enumerated in amd/include/kgd_pp_interface.h under the
 774  * 'amd_pp_sensors' enumeration.  For instance to read the UVD VCLK
 775  * you would use the offset 3 * 4 = 12.
 776  */
 777 static ssize_t amdgpu_debugfs_sensor_read(struct file *f, char __user *buf,
 778                                         size_t size, loff_t *pos)
 779 {
 780         struct amdgpu_device *adev = file_inode(f)->i_private;
 781         int idx, x, outsize, r, valuesize;
 782         uint32_t values[16];
 783
 784         if (size & 3 || *pos & 0x3)
 785                 return -EINVAL;
 786
 787         if (!adev->pm.dpm_enabled)
 788                 return -EINVAL;
 789
 790         /* convert offset to sensor number */
 791         idx = *pos >> 2;
 792
 793         valuesize = sizeof(values);
 794
 795         r = pm_runtime_get_sync(adev->ddev->dev);
 796         if (r < 0) {
 797                 pm_runtime_put_autosuspend(adev->ddev->dev);
 798                 return r;
 799         }
 800
 801         r = amdgpu_virt_enable_access_debugfs(adev);
 802         if (r < 0) {
 803                 pm_runtime_put_autosuspend(adev->ddev->dev);
 804                 return r;
 805         }
 806
 807         r = amdgpu_dpm_read_sensor(adev, idx, &values[0], &valuesize);
 808
 809         pm_runtime_mark_last_busy(adev->ddev->dev);
 810         pm_runtime_put_autosuspend(adev->ddev->dev);
 811
 812         if (r) {
 813                 amdgpu_virt_disable_access_debugfs(adev);
 814                 return r;
 815         }
 816
 817         if (size > valuesize) {
 818                 amdgpu_virt_disable_access_debugfs(adev);
 819                 return -EINVAL;
 820         }
 821
 822         outsize = 0;
 823         x = 0;
 824         if (!r) {
 825                 while (size) {
 826                         r = put_user(values[x++], (int32_t *)buf);
 827                         buf += 4;
 828                         size -= 4;
 829                         outsize += 4;
 830                 }
 831         }
 832
 833         amdgpu_virt_disable_access_debugfs(adev);
 834         return !r ? outsize : r;
 835 }
 836
 837 /** amdgpu_debugfs_wave_read - Read WAVE STATUS data
 838  *
 839  * @f: open file handle
 840  * @buf: User buffer to store read data in
 841  * @size: Number of bytes to read
 842  * @pos:  Offset to seek to
 843  *
 844  * The offset being sought changes which wave that the status data
 845  * will be returned for.  The bits are used as follows:
 846  *
 847  * Bits 0..6:   Byte offset into data
 848  * Bits 7..14:  SE selector
 849  * Bits 15..22: SH/SA selector
 850  * Bits 23..30: CU/{WGP+SIMD} selector
 851  * Bits 31..36: WAVE ID selector
 852  * Bits 37..44: SIMD ID selector
 853  *
 854  * The returned data begins with one DWORD of version information
 855  * Followed by WAVE STATUS registers relevant to the GFX IP version
 856  * being used.  See gfx_v8_0_read_wave_data() for an example output.
 857  */
 858 static ssize_t amdgpu_debugfs_wave_read(struct file *f, char __user *buf,
 859                                         size_t size, loff_t *pos)
 860 {
 861         struct amdgpu_device *adev = f->f_inode->i_private;
 862         int r, x;
 863         ssize_t result=0;
 864         uint32_t offset, se, sh, cu, wave, simd, data[32];
 865
 866         if (size & 3 || *pos & 3)
 867                 return -EINVAL;
 868
 869         /* decode offset */
 870         offset = (*pos & GENMASK_ULL(6, 0));
 871         se = (*pos & GENMASK_ULL(14, 7)) >> 7;
 872         sh = (*pos & GENMASK_ULL(22, 15)) >> 15;
 873         cu = (*pos & GENMASK_ULL(30, 23)) >> 23;
 874         wave = (*pos & GENMASK_ULL(36, 31)) >> 31;
 875         simd = (*pos & GENMASK_ULL(44, 37)) >> 37;
 876
 877         r = pm_runtime_get_sync(adev->ddev->dev);
 878         if (r < 0) {
 879                 pm_runtime_put_autosuspend(adev->ddev->dev);
 880                 return r;
 881         }
 882
 883         r = amdgpu_virt_enable_access_debugfs(adev);
 884         if (r < 0) {
 885                 pm_runtime_put_autosuspend(adev->ddev->dev);
 886                 return r;
 887         }
 888
 889         /* switch to the specific se/sh/cu */
 890         mutex_lock(&adev->grbm_idx_mutex);
 891         amdgpu_gfx_select_se_sh(adev, se, sh, cu);
 892
 893         x = 0;
 894         if (adev->gfx.funcs->read_wave_data)
 895                 adev->gfx.funcs->read_wave_data(adev, simd, wave, data, &x);
 896
 897         amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF);
 898         mutex_unlock(&adev->grbm_idx_mutex);
 899
 900         pm_runtime_mark_last_busy(adev->ddev->dev);
 901         pm_runtime_put_autosuspend(adev->ddev->dev);
 902
 903         if (!x) {
 904                 amdgpu_virt_disable_access_debugfs(adev);
 905                 return -EINVAL;
 906         }
 907
 908         while (size && (offset < x * 4)) {
 909                 uint32_t value;
 910
 911                 value = data[offset >> 2];
 912                 r = put_user(value, (uint32_t *)buf);
 913                 if (r) {
 914                         amdgpu_virt_disable_access_debugfs(adev);
 915                         return r;
 916                 }
 917
 918                 result += 4;
 919                 buf += 4;
 920                 offset += 4;
 921                 size -= 4;
 922         }
 923
 924         amdgpu_virt_disable_access_debugfs(adev);
 925         return result;
 926 }
 927
 928 /** amdgpu_debugfs_gpr_read - Read wave gprs
 929  *
 930  * @f: open file handle
 931  * @buf: User buffer to store read data in
 932  * @size: Number of bytes to read
 933  * @pos:  Offset to seek to
 934  *
 935  * The offset being sought changes which wave that the status data
 936  * will be returned for.  The bits are used as follows:
 937  *
 938  * Bits 0..11:  Byte offset into data
 939  * Bits 12..19: SE selector
 940  * Bits 20..27: SH/SA selector
 941  * Bits 28..35: CU/{WGP+SIMD} selector
 942  * Bits 36..43: WAVE ID selector
 943  * Bits 37..44: SIMD ID selector
 944  * Bits 52..59: Thread selector
 945  * Bits 60..61: Bank selector (VGPR=0,SGPR=1)
 946  *
 947  * The return data comes from the SGPR or VGPR register bank for
 948  * the selected operational unit.
 949  */
 950 static ssize_t amdgpu_debugfs_gpr_read(struct file *f, char __user *buf,
 951                                         size_t size, loff_t *pos)
 952 {
 953         struct amdgpu_device *adev = f->f_inode->i_private;
 954         int r;
 955         ssize_t result = 0;
 956         uint32_t offset, se, sh, cu, wave, simd, thread, bank, *data;
 957
 958         if (size > 4096 || size & 3 || *pos & 3)
 959                 return -EINVAL;
 960
 961         /* decode offset */
 962         offset = (*pos & GENMASK_ULL(11, 0)) >> 2;
 963         se = (*pos & GENMASK_ULL(19, 12)) >> 12;
 964         sh = (*pos & GENMASK_ULL(27, 20)) >> 20;
 965         cu = (*pos & GENMASK_ULL(35, 28)) >> 28;
 966         wave = (*pos & GENMASK_ULL(43, 36)) >> 36;
 967         simd = (*pos & GENMASK_ULL(51, 44)) >> 44;
 968         thread = (*pos & GENMASK_ULL(59, 52)) >> 52;
 969         bank = (*pos & GENMASK_ULL(61, 60)) >> 60;
 970
 971         data = kcalloc(1024, sizeof(*data), GFP_KERNEL);
 972         if (!data)
 973                 return -ENOMEM;
 974
 975         r = pm_runtime_get_sync(adev->ddev->dev);
 976         if (r < 0)
 977                 goto err;
 978
 979         r = amdgpu_virt_enable_access_debugfs(adev);
 980         if (r < 0)
 981                 goto err;
 982
 983         /* switch to the specific se/sh/cu */
 984         mutex_lock(&adev->grbm_idx_mutex);
 985         amdgpu_gfx_select_se_sh(adev, se, sh, cu);
 986
 987         if (bank == 0) {
 988                 if (adev->gfx.funcs->read_wave_vgprs)
 989                         adev->gfx.funcs->read_wave_vgprs(adev, simd, wave, thread, offset, size>>2, data);
 990         } else {
 991                 if (adev->gfx.funcs->read_wave_sgprs)
 992                         adev->gfx.funcs->read_wave_sgprs(adev, simd, wave, offset, size>>2, data);
 993         }
 994
 995         amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF);
 996         mutex_unlock(&adev->grbm_idx_mutex);
 997
 998         pm_runtime_mark_last_busy(adev->ddev->dev);
 999         pm_runtime_put_autosuspend(adev->ddev->dev);
1000
1001         while (size) {
1002                 uint32_t value;
1003
1004                 value = data[result >> 2];
1005                 r = put_user(value, (uint32_t *)buf);
1006                 if (r) {
1007                         amdgpu_virt_disable_access_debugfs(adev);
1008                         goto err;
1009                 }
1010
1011                 result += 4;
1012                 buf += 4;
1013                 size -= 4;
1014         }
1015
1016         kfree(data);
1017         amdgpu_virt_disable_access_debugfs(adev);
1018         return result;
1019
1020 err:
1021         pm_runtime_put_autosuspend(adev->ddev->dev);
1022         kfree(data);
1023         return r;
1024 }
1025
1026 /**
1027  * amdgpu_debugfs_regs_gfxoff_write - Enable/disable GFXOFF
1028  *
1029  * @f: open file handle
1030  * @buf: User buffer to write data from
1031  * @size: Number of bytes to write
1032  * @pos:  Offset to seek to
1033  *
1034  * Write a 32-bit zero to disable or a 32-bit non-zero to enable
1035  */
1036 static ssize_t amdgpu_debugfs_gfxoff_write(struct file *f, const char __user *buf,
1037                                          size_t size, loff_t *pos)
1038 {
1039         struct amdgpu_device *adev = file_inode(f)->i_private;
1040         ssize_t result = 0;
1041         int r;
1042
1043         if (size & 0x3 || *pos & 0x3)
1044                 return -EINVAL;
1045
1046         r = pm_runtime_get_sync(adev->ddev->dev);
1047         if (r < 0) {
1048                 pm_runtime_put_autosuspend(adev->ddev->dev);
1049                 return r;
1050         }
1051
1052         while (size) {
1053                 uint32_t value;
1054
1055                 r = get_user(value, (uint32_t *)buf);
1056                 if (r) {
1057                         pm_runtime_mark_last_busy(adev->ddev->dev);
1058                         pm_runtime_put_autosuspend(adev->ddev->dev);
1059                         return r;
1060                 }
1061
1062                 amdgpu_gfx_off_ctrl(adev, value ? true : false);
1063
1064                 result += 4;
1065                 buf += 4;
1066                 *pos += 4;
1067                 size -= 4;
1068         }
1069
1070         pm_runtime_mark_last_busy(adev->ddev->dev);
1071         pm_runtime_put_autosuspend(adev->ddev->dev);
1072
1073         return result;
1074 }
1075
1076
1077 /**
1078  * amdgpu_debugfs_regs_gfxoff_status - read gfxoff status
1079  *
1080  * @f: open file handle
1081  * @buf: User buffer to store read data in
1082  * @size: Number of bytes to read
1083  * @pos:  Offset to seek to
1084  */
1085 static ssize_t amdgpu_debugfs_gfxoff_read(struct file *f, char __user *buf,
1086                                          size_t size, loff_t *pos)
1087 {
1088         struct amdgpu_device *adev = file_inode(f)->i_private;
1089         ssize_t result = 0;
1090         int r;
1091
1092         if (size & 0x3 || *pos & 0x3)
1093                 return -EINVAL;
1094
1095         r = pm_runtime_get_sync(adev->ddev->dev);
1096         if (r < 0)
1097                 return r;
1098
1099         while (size) {
1100                 uint32_t value;
1101
1102                 r = amdgpu_get_gfx_off_status(adev, &value);
1103                 if (r) {
1104                         pm_runtime_mark_last_busy(adev->ddev->dev);
1105                         pm_runtime_put_autosuspend(adev->ddev->dev);
1106                         return r;
1107                 }
1108
1109                 r = put_user(value, (uint32_t *)buf);
1110                 if (r) {
1111                         pm_runtime_mark_last_busy(adev->ddev->dev);
1112                         pm_runtime_put_autosuspend(adev->ddev->dev);
1113                         return r;
1114                 }
1115
1116                 result += 4;
1117                 buf += 4;
1118                 *pos += 4;
1119                 size -= 4;
1120         }
1121
1122         pm_runtime_mark_last_busy(adev->ddev->dev);
1123         pm_runtime_put_autosuspend(adev->ddev->dev);
1124
1125         return result;
1126 }
1127
1128 static const struct file_operations amdgpu_debugfs_regs_fops = {
1129         .owner = THIS_MODULE,
1130         .read = amdgpu_debugfs_regs_read,
1131         .write = amdgpu_debugfs_regs_write,
1132         .llseek = default_llseek
1133 };
1134 static const struct file_operations amdgpu_debugfs_regs_didt_fops = {
1135         .owner = THIS_MODULE,
1136         .read = amdgpu_debugfs_regs_didt_read,
1137         .write = amdgpu_debugfs_regs_didt_write,
1138         .llseek = default_llseek
1139 };
1140 static const struct file_operations amdgpu_debugfs_regs_pcie_fops = {
1141         .owner = THIS_MODULE,
1142         .read = amdgpu_debugfs_regs_pcie_read,
1143         .write = amdgpu_debugfs_regs_pcie_write,
1144         .llseek = default_llseek
1145 };
1146 static const struct file_operations amdgpu_debugfs_regs_smc_fops = {
1147         .owner = THIS_MODULE,
1148         .read = amdgpu_debugfs_regs_smc_read,
1149         .write = amdgpu_debugfs_regs_smc_write,
1150         .llseek = default_llseek
1151 };
1152
1153 static const struct file_operations amdgpu_debugfs_gca_config_fops = {
1154         .owner = THIS_MODULE,
1155         .read = amdgpu_debugfs_gca_config_read,
1156         .llseek = default_llseek
1157 };
1158
1159 static const struct file_operations amdgpu_debugfs_sensors_fops = {
1160         .owner = THIS_MODULE,
1161         .read = amdgpu_debugfs_sensor_read,
1162         .llseek = default_llseek
1163 };
1164
1165 static const struct file_operations amdgpu_debugfs_wave_fops = {
1166         .owner = THIS_MODULE,
1167         .read = amdgpu_debugfs_wave_read,
1168         .llseek = default_llseek
1169 };
1170 static const struct file_operations amdgpu_debugfs_gpr_fops = {
1171         .owner = THIS_MODULE,
1172         .read = amdgpu_debugfs_gpr_read,
1173         .llseek = default_llseek
1174 };
1175
1176 static const struct file_operations amdgpu_debugfs_gfxoff_fops = {
1177         .owner = THIS_MODULE,
1178         .read = amdgpu_debugfs_gfxoff_read,
1179         .write = amdgpu_debugfs_gfxoff_write,
1180         .llseek = default_llseek
1181 };
1182
1183 static const struct file_operations *debugfs_regs[] = {
1184         &amdgpu_debugfs_regs_fops,
1185         &amdgpu_debugfs_regs_didt_fops,
1186         &amdgpu_debugfs_regs_pcie_fops,
1187         &amdgpu_debugfs_regs_smc_fops,
1188         &amdgpu_debugfs_gca_config_fops,
1189         &amdgpu_debugfs_sensors_fops,
1190         &amdgpu_debugfs_wave_fops,
1191         &amdgpu_debugfs_gpr_fops,
1192         &amdgpu_debugfs_gfxoff_fops,
1193 };
1194
1195 static const char *debugfs_regs_names[] = {
1196         "amdgpu_regs",
1197         "amdgpu_regs_didt",
1198         "amdgpu_regs_pcie",
1199         "amdgpu_regs_smc",
1200         "amdgpu_gca_config",
1201         "amdgpu_sensors",
1202         "amdgpu_wave",
1203         "amdgpu_gpr",
1204         "amdgpu_gfxoff",
1205 };
1206
1207 /**
1208  * amdgpu_debugfs_regs_init -   Initialize debugfs entries that provide
1209  *                                                              register access.
1210  *
1211  * @adev: The device to attach the debugfs entries to
1212  */
1213 int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
1214 {
1215         struct drm_minor *minor = adev->ddev->primary;
1216         struct dentry *ent, *root = minor->debugfs_root;
1217         unsigned int i;
1218
1219         for (i = 0; i < ARRAY_SIZE(debugfs_regs); i++) {
1220                 ent = debugfs_create_file(debugfs_regs_names[i],
1221                                           S_IFREG | S_IRUGO, root,
1222                                           adev, debugfs_regs[i]);
1223                 if (!i && !IS_ERR_OR_NULL(ent))
1224                         i_size_write(ent->d_inode, adev->rmmio_size);
1225                 adev->debugfs_regs[i] = ent;
1226         }
1227
1228         return 0;
1229 }
1230
1231 static int amdgpu_debugfs_test_ib(struct seq_file *m, void *data)
1232 {
1233         struct drm_info_node *node = (struct drm_info_node *) m->private;
1234         struct drm_device *dev = node->minor->dev;
1235         struct amdgpu_device *adev = dev->dev_private;
1236         int r = 0, i;
1237
1238         r = pm_runtime_get_sync(dev->dev);
1239         if (r < 0) {
1240                 pm_runtime_put_autosuspend(adev->ddev->dev);
1241                 return r;
1242         }
1243
1244         /* Avoid accidently unparking the sched thread during GPU reset */
1245         mutex_lock(&adev->lock_reset);
1246
1247         /* hold on the scheduler */
1248         for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
1249                 struct amdgpu_ring *ring = adev->rings[i];
1250
1251                 if (!ring || !ring->sched.thread)
1252                         continue;
1253                 kthread_park(ring->sched.thread);
1254         }
1255
1256         seq_printf(m, "run ib test:\n");
1257         r = amdgpu_ib_ring_tests(adev);
1258         if (r)
1259                 seq_printf(m, "ib ring tests failed (%d).\n", r);
1260         else
1261                 seq_printf(m, "ib ring tests passed.\n");
1262
1263         /* go on the scheduler */
1264         for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
1265                 struct amdgpu_ring *ring = adev->rings[i];
1266
1267                 if (!ring || !ring->sched.thread)
1268                         continue;
1269                 kthread_unpark(ring->sched.thread);
1270         }
1271
1272         mutex_unlock(&adev->lock_reset);
1273
1274         pm_runtime_mark_last_busy(dev->dev);
1275         pm_runtime_put_autosuspend(dev->dev);
1276
1277         return 0;
1278 }
1279
1280 static int amdgpu_debugfs_get_vbios_dump(struct seq_file *m, void *data)
1281 {
1282         struct drm_info_node *node = (struct drm_info_node *) m->private;
1283         struct drm_device *dev = node->minor->dev;
1284         struct amdgpu_device *adev = dev->dev_private;
1285
1286         seq_write(m, adev->bios, adev->bios_size);
1287         return 0;
1288 }
1289
1290 static int amdgpu_debugfs_evict_vram(struct seq_file *m, void *data)
1291 {
1292         struct drm_info_node *node = (struct drm_info_node *)m->private;
1293         struct drm_device *dev = node->minor->dev;
1294         struct amdgpu_device *adev = dev->dev_private;
1295         int r;
1296
1297         r = pm_runtime_get_sync(dev->dev);
1298         if (r < 0) {
1299                 pm_runtime_put_autosuspend(adev->ddev->dev);
1300                 return r;
1301         }
1302
1303         seq_printf(m, "(%d)\n", amdgpu_bo_evict_vram(adev));
1304
1305         pm_runtime_mark_last_busy(dev->dev);
1306         pm_runtime_put_autosuspend(dev->dev);
1307
1308         return 0;
1309 }
1310
1311 static int amdgpu_debugfs_evict_gtt(struct seq_file *m, void *data)
1312 {
1313         struct drm_info_node *node = (struct drm_info_node *)m->private;
1314         struct drm_device *dev = node->minor->dev;
1315         struct amdgpu_device *adev = dev->dev_private;
1316         int r;
1317
1318         r = pm_runtime_get_sync(dev->dev);
1319         if (r < 0) {
1320                 pm_runtime_put_autosuspend(adev->ddev->dev);
1321                 return r;
1322         }
1323
1324         seq_printf(m, "(%d)\n", ttm_bo_evict_mm(&adev->mman.bdev, TTM_PL_TT));
1325
1326         pm_runtime_mark_last_busy(dev->dev);
1327         pm_runtime_put_autosuspend(dev->dev);
1328
1329         return 0;
1330 }
1331
1332 static const struct drm_info_list amdgpu_debugfs_list[] = {
1333         {"amdgpu_vbios", amdgpu_debugfs_get_vbios_dump},
1334         {"amdgpu_test_ib", &amdgpu_debugfs_test_ib},
1335         {"amdgpu_evict_vram", &amdgpu_debugfs_evict_vram},
1336         {"amdgpu_evict_gtt", &amdgpu_debugfs_evict_gtt},
1337 };
1338
1339 static void amdgpu_ib_preempt_fences_swap(struct amdgpu_ring *ring,
1340                                           struct dma_fence **fences)
1341 {
1342         struct amdgpu_fence_driver *drv = &ring->fence_drv;
1343         uint32_t sync_seq, last_seq;
1344
1345         last_seq = atomic_read(&ring->fence_drv.last_seq);
1346         sync_seq = ring->fence_drv.sync_seq;
1347
1348         last_seq &= drv->num_fences_mask;
1349         sync_seq &= drv->num_fences_mask;
1350
1351         do {
1352                 struct dma_fence *fence, **ptr;
1353
1354                 ++last_seq;
1355                 last_seq &= drv->num_fences_mask;
1356                 ptr = &drv->fences[last_seq];
1357
1358                 fence = rcu_dereference_protected(*ptr, 1);
1359                 RCU_INIT_POINTER(*ptr, NULL);
1360
1361                 if (!fence)
1362                         continue;
1363
1364                 fences[last_seq] = fence;
1365
1366         } while (last_seq != sync_seq);
1367 }
1368
1369 static void amdgpu_ib_preempt_signal_fences(struct dma_fence **fences,
1370                                             int length)
1371 {
1372         int i;
1373         struct dma_fence *fence;
1374
1375         for (i = 0; i < length; i++) {
1376                 fence = fences[i];
1377                 if (!fence)
1378                         continue;
1379                 dma_fence_signal(fence);
1380                 dma_fence_put(fence);
1381         }
1382 }
1383
1384 static void amdgpu_ib_preempt_job_recovery(struct drm_gpu_scheduler *sched)
1385 {
1386         struct drm_sched_job *s_job;
1387         struct dma_fence *fence;
1388
1389         spin_lock(&sched->job_list_lock);
1390         list_for_each_entry(s_job, &sched->ring_mirror_list, node) {
1391                 fence = sched->ops->run_job(s_job);
1392                 dma_fence_put(fence);
1393         }
1394         spin_unlock(&sched->job_list_lock);
1395 }
1396
1397 static void amdgpu_ib_preempt_mark_partial_job(struct amdgpu_ring *ring)
1398 {
1399         struct amdgpu_job *job;
1400         struct drm_sched_job *s_job, *tmp;
1401         uint32_t preempt_seq;
1402         struct dma_fence *fence, **ptr;
1403         struct amdgpu_fence_driver *drv = &ring->fence_drv;
1404         struct drm_gpu_scheduler *sched = &ring->sched;
1405         bool preempted = true;
1406
1407         if (ring->funcs->type != AMDGPU_RING_TYPE_GFX)
1408                 return;
1409
1410         preempt_seq = le32_to_cpu(*(drv->cpu_addr + 2));
1411         if (preempt_seq <= atomic_read(&drv->last_seq)) {
1412                 preempted = false;
1413                 goto no_preempt;
1414         }
1415
1416         preempt_seq &= drv->num_fences_mask;
1417         ptr = &drv->fences[preempt_seq];
1418         fence = rcu_dereference_protected(*ptr, 1);
1419
1420 no_preempt:
1421         spin_lock(&sched->job_list_lock);
1422         list_for_each_entry_safe(s_job, tmp, &sched->ring_mirror_list, node) {
1423                 if (dma_fence_is_signaled(&s_job->s_fence->finished)) {
1424                         /* remove job from ring_mirror_list */
1425                         list_del_init(&s_job->node);
1426                         sched->ops->free_job(s_job);
1427                         continue;
1428                 }
1429                 job = to_amdgpu_job(s_job);
1430                 if (preempted && job->fence == fence)
1431                         /* mark the job as preempted */
1432                         job->preemption_status |= AMDGPU_IB_PREEMPTED;
1433         }
1434         spin_unlock(&sched->job_list_lock);
1435 }
1436
1437 static int amdgpu_debugfs_ib_preempt(void *data, u64 val)
1438 {
1439         int r, resched, length;
1440         struct amdgpu_ring *ring;
1441         struct dma_fence **fences = NULL;
1442         struct amdgpu_device *adev = (struct amdgpu_device *)data;
1443
1444         if (val >= AMDGPU_MAX_RINGS)
1445                 return -EINVAL;
1446
1447         ring = adev->rings[val];
1448
1449         if (!ring || !ring->funcs->preempt_ib || !ring->sched.thread)
1450                 return -EINVAL;
1451
1452         /* the last preemption failed */
1453         if (ring->trail_seq != le32_to_cpu(*ring->trail_fence_cpu_addr))
1454                 return -EBUSY;
1455
1456         length = ring->fence_drv.num_fences_mask + 1;
1457         fences = kcalloc(length, sizeof(void *), GFP_KERNEL);
1458         if (!fences)
1459                 return -ENOMEM;
1460
1461         /* Avoid accidently unparking the sched thread during GPU reset */
1462         mutex_lock(&adev->lock_reset);
1463
1464         /* stop the scheduler */
1465         kthread_park(ring->sched.thread);
1466
1467         resched = ttm_bo_lock_delayed_workqueue(&adev->mman.bdev);
1468
1469         /* preempt the IB */
1470         r = amdgpu_ring_preempt_ib(ring);
1471         if (r) {
1472                 DRM_WARN("failed to preempt ring %d\n", ring->idx);
1473                 goto failure;
1474         }
1475
1476         amdgpu_fence_process(ring);
1477
1478         if (atomic_read(&ring->fence_drv.last_seq) !=
1479             ring->fence_drv.sync_seq) {
1480                 DRM_INFO("ring %d was preempted\n", ring->idx);
1481
1482                 amdgpu_ib_preempt_mark_partial_job(ring);
1483
1484                 /* swap out the old fences */
1485                 amdgpu_ib_preempt_fences_swap(ring, fences);
1486
1487                 amdgpu_fence_driver_force_completion(ring);
1488
1489                 /* resubmit unfinished jobs */
1490                 amdgpu_ib_preempt_job_recovery(&ring->sched);
1491
1492                 /* wait for jobs finished */
1493                 amdgpu_fence_wait_empty(ring);
1494
1495                 /* signal the old fences */
1496                 amdgpu_ib_preempt_signal_fences(fences, length);
1497         }
1498
1499 failure:
1500         /* restart the scheduler */
1501         kthread_unpark(ring->sched.thread);
1502
1503         mutex_unlock(&adev->lock_reset);
1504
1505         ttm_bo_unlock_delayed_workqueue(&adev->mman.bdev, resched);
1506
1507         kfree(fences);
1508
1509         return 0;
1510 }
1511
1512 static int amdgpu_debugfs_sclk_set(void *data, u64 val)
1513 {
1514         int ret = 0;
1515         uint32_t max_freq, min_freq;
1516         struct amdgpu_device *adev = (struct amdgpu_device *)data;
1517
1518         if (amdgpu_sriov_vf(adev) && !amdgpu_sriov_is_pp_one_vf(adev))
1519                 return -EINVAL;
1520
1521         ret = pm_runtime_get_sync(adev->ddev->dev);
1522         if (ret < 0) {
1523                 pm_runtime_put_autosuspend(adev->ddev->dev);
1524                 return ret;
1525         }
1526
1527         if (is_support_sw_smu(adev)) {
1528                 ret = smu_get_dpm_freq_range(&adev->smu, SMU_SCLK, &min_freq, &max_freq);
1529                 if (ret || val > max_freq || val < min_freq)
1530                         return -EINVAL;
1531                 ret = smu_set_soft_freq_range(&adev->smu, SMU_SCLK, (uint32_t)val, (uint32_t)val);
1532         } else {
1533                 return 0;
1534         }
1535
1536         pm_runtime_mark_last_busy(adev->ddev->dev);
1537         pm_runtime_put_autosuspend(adev->ddev->dev);
1538
1539         if (ret)
1540                 return -EINVAL;
1541
1542         return 0;
1543 }
1544
1545 DEFINE_SIMPLE_ATTRIBUTE(fops_ib_preempt, NULL,
1546                         amdgpu_debugfs_ib_preempt, "%llu\n");
1547
1548 DEFINE_SIMPLE_ATTRIBUTE(fops_sclk_set, NULL,
1549                         amdgpu_debugfs_sclk_set, "%llu\n");
1550
1551 int amdgpu_debugfs_init(struct amdgpu_device *adev)
1552 {
1553         int r, i;
1554
1555         adev->debugfs_preempt =
1556                 debugfs_create_file("amdgpu_preempt_ib", 0600,
1557                                     adev->ddev->primary->debugfs_root, adev,
1558                                     &fops_ib_preempt);
1559         if (!(adev->debugfs_preempt)) {
1560                 DRM_ERROR("unable to create amdgpu_preempt_ib debugsfs file\n");
1561                 return -EIO;
1562         }
1563
1564         adev->smu.debugfs_sclk =
1565                 debugfs_create_file("amdgpu_force_sclk", 0200,
1566                                     adev->ddev->primary->debugfs_root, adev,
1567                                     &fops_sclk_set);
1568         if (!(adev->smu.debugfs_sclk)) {
1569                 DRM_ERROR("unable to create amdgpu_set_sclk debugsfs file\n");
1570                 return -EIO;
1571         }
1572
1573         /* Register debugfs entries for amdgpu_ttm */
1574         r = amdgpu_ttm_debugfs_init(adev);
1575         if (r) {
1576                 DRM_ERROR("Failed to init debugfs\n");
1577                 return r;
1578         }
1579
1580         r = amdgpu_debugfs_pm_init(adev);
1581         if (r) {
1582                 DRM_ERROR("Failed to register debugfs file for dpm!\n");
1583                 return r;
1584         }
1585
1586         if (amdgpu_debugfs_sa_init(adev)) {
1587                 dev_err(adev->dev, "failed to register debugfs file for SA\n");
1588         }
1589
1590         if (amdgpu_debugfs_fence_init(adev))
1591                 dev_err(adev->dev, "fence debugfs file creation failed\n");
1592
1593         r = amdgpu_debugfs_gem_init(adev);
1594         if (r)
1595                 DRM_ERROR("registering gem debugfs failed (%d).\n", r);
1596
1597         r = amdgpu_debugfs_regs_init(adev);
1598         if (r)
1599                 DRM_ERROR("registering register debugfs failed (%d).\n", r);
1600
1601         r = amdgpu_debugfs_firmware_init(adev);
1602         if (r)
1603                 DRM_ERROR("registering firmware debugfs failed (%d).\n", r);
1604
1605 #if defined(CONFIG_DRM_AMD_DC)
1606         if (amdgpu_device_has_dc_support(adev)) {
1607                 if (dtn_debugfs_init(adev))
1608                         DRM_ERROR("amdgpu: failed initialize dtn debugfs support.\n");
1609         }
1610 #endif
1611
1612         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
1613                 struct amdgpu_ring *ring = adev->rings[i];
1614
1615                 if (!ring)
1616                         continue;
1617
1618                 if (amdgpu_debugfs_ring_init(adev, ring)) {
1619                         DRM_ERROR("Failed to register debugfs file for rings !\n");
1620                 }
1621         }
1622
1623         amdgpu_ras_debugfs_create_all(adev);
1624
1625         amdgpu_debugfs_autodump_init(adev);
1626
1627         amdgpu_rap_debugfs_init(adev);
1628
1629         return amdgpu_debugfs_add_files(adev, amdgpu_debugfs_list,
1630                                         ARRAY_SIZE(amdgpu_debugfs_list));
1631 }
1632
1633 #else
1634 int amdgpu_debugfs_init(struct amdgpu_device *adev)
1635 {
1636         return 0;
1637 }
1638 int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
1639 {
1640         return 0;
1641 }
1642 #endif