drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c

   1 /*
   2  * Copyright 2008 Advanced Micro Devices, Inc.
   3  * Copyright 2008 Red Hat Inc.
   4  * Copyright 2009 Jerome Glisse.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22  * OTHER DEALINGS IN THE SOFTWARE.
  23  *
  24  */
  25
  26 #include <linux/kthread.h>
  27 #include <linux/pci.h>
  28 #include <linux/uaccess.h>
  29 #include <linux/pm_runtime.h>
  30 #include <linux/poll.h>
  31
  32 #include "amdgpu.h"
  33 #include "amdgpu_pm.h"
  34 #include "amdgpu_dm_debugfs.h"
  35 #include "amdgpu_ras.h"
  36 #include "amdgpu_rap.h"
  37 #include "amdgpu_securedisplay.h"
  38 #include "amdgpu_fw_attestation.h"
  39
  40 int amdgpu_debugfs_wait_dump(struct amdgpu_device *adev)
  41 {
  42 #if defined(CONFIG_DEBUG_FS)
  43         unsigned long timeout = 600 * HZ;
  44         int ret;
  45
  46         wake_up_interruptible(&adev->autodump.gpu_hang);
  47
  48         ret = wait_for_completion_interruptible_timeout(&adev->autodump.dumping, timeout);
  49         if (ret == 0) {
  50                 pr_err("autodump: timeout, move on to gpu recovery\n");
  51                 return -ETIMEDOUT;
  52         }
  53 #endif
  54         return 0;
  55 }
  56
  57 #if defined(CONFIG_DEBUG_FS)
  58
  59 static int amdgpu_debugfs_autodump_open(struct inode *inode, struct file *file)
  60 {
  61         struct amdgpu_device *adev = inode->i_private;
  62         int ret;
  63
  64         file->private_data = adev;
  65
  66         ret = down_read_killable(&adev->reset_sem);
  67         if (ret)
  68                 return ret;
  69
  70         if (adev->autodump.dumping.done) {
  71                 reinit_completion(&adev->autodump.dumping);
  72                 ret = 0;
  73         } else {
  74                 ret = -EBUSY;
  75         }
  76
  77         up_read(&adev->reset_sem);
  78
  79         return ret;
  80 }
  81
  82 static int amdgpu_debugfs_autodump_release(struct inode *inode, struct file *file)
  83 {
  84         struct amdgpu_device *adev = file->private_data;
  85
  86         complete_all(&adev->autodump.dumping);
  87         return 0;
  88 }
  89
  90 static unsigned int amdgpu_debugfs_autodump_poll(struct file *file, struct poll_table_struct *poll_table)
  91 {
  92         struct amdgpu_device *adev = file->private_data;
  93
  94         poll_wait(file, &adev->autodump.gpu_hang, poll_table);
  95
  96         if (amdgpu_in_reset(adev))
  97                 return POLLIN | POLLRDNORM | POLLWRNORM;
  98
  99         return 0;
 100 }
 101
 102 static const struct file_operations autodump_debug_fops = {
 103         .owner = THIS_MODULE,
 104         .open = amdgpu_debugfs_autodump_open,
 105         .poll = amdgpu_debugfs_autodump_poll,
 106         .release = amdgpu_debugfs_autodump_release,
 107 };
 108
 109 static void amdgpu_debugfs_autodump_init(struct amdgpu_device *adev)
 110 {
 111         init_completion(&adev->autodump.dumping);
 112         complete_all(&adev->autodump.dumping);
 113         init_waitqueue_head(&adev->autodump.gpu_hang);
 114
 115         debugfs_create_file("amdgpu_autodump", 0600,
 116                 adev_to_drm(adev)->primary->debugfs_root,
 117                 adev, &autodump_debug_fops);
 118 }
 119
 120 /**
 121  * amdgpu_debugfs_process_reg_op - Handle MMIO register reads/writes
 122  *
 123  * @read: True if reading
 124  * @f: open file handle
 125  * @buf: User buffer to write/read to
 126  * @size: Number of bytes to write/read
 127  * @pos:  Offset to seek to
 128  *
 129  * This debugfs entry has special meaning on the offset being sought.
 130  * Various bits have different meanings:
 131  *
 132  * Bit 62:  Indicates a GRBM bank switch is needed
 133  * Bit 61:  Indicates a SRBM bank switch is needed (implies bit 62 is
 134  *          zero)
 135  * Bits 24..33: The SE or ME selector if needed
 136  * Bits 34..43: The SH (or SA) or PIPE selector if needed
 137  * Bits 44..53: The INSTANCE (or CU/WGP) or QUEUE selector if needed
 138  *
 139  * Bit 23:  Indicates that the PM power gating lock should be held
 140  *          This is necessary to read registers that might be
 141  *          unreliable during a power gating transistion.
 142  *
 143  * The lower bits are the BYTE offset of the register to read.  This
 144  * allows reading multiple registers in a single call and having
 145  * the returned size reflect that.
 146  */
 147 static int  amdgpu_debugfs_process_reg_op(bool read, struct file *f,
 148                 char __user *buf, size_t size, loff_t *pos)
 149 {
 150         struct amdgpu_device *adev = file_inode(f)->i_private;
 151         ssize_t result = 0;
 152         int r;
 153         bool pm_pg_lock, use_bank, use_ring;
 154         unsigned instance_bank, sh_bank, se_bank, me, pipe, queue, vmid;
 155
 156         pm_pg_lock = use_bank = use_ring = false;
 157         instance_bank = sh_bank = se_bank = me = pipe = queue = vmid = 0;
 158
 159         if (size & 0x3 || *pos & 0x3 ||
 160                         ((*pos & (1ULL << 62)) && (*pos & (1ULL << 61))))
 161                 return -EINVAL;
 162
 163         /* are we reading registers for which a PG lock is necessary? */
 164         pm_pg_lock = (*pos >> 23) & 1;
 165
 166         if (*pos & (1ULL << 62)) {
 167                 se_bank = (*pos & GENMASK_ULL(33, 24)) >> 24;
 168                 sh_bank = (*pos & GENMASK_ULL(43, 34)) >> 34;
 169                 instance_bank = (*pos & GENMASK_ULL(53, 44)) >> 44;
 170
 171                 if (se_bank == 0x3FF)
 172                         se_bank = 0xFFFFFFFF;
 173                 if (sh_bank == 0x3FF)
 174                         sh_bank = 0xFFFFFFFF;
 175                 if (instance_bank == 0x3FF)
 176                         instance_bank = 0xFFFFFFFF;
 177                 use_bank = true;
 178         } else if (*pos & (1ULL << 61)) {
 179
 180                 me = (*pos & GENMASK_ULL(33, 24)) >> 24;
 181                 pipe = (*pos & GENMASK_ULL(43, 34)) >> 34;
 182                 queue = (*pos & GENMASK_ULL(53, 44)) >> 44;
 183                 vmid = (*pos & GENMASK_ULL(58, 54)) >> 54;
 184
 185                 use_ring = true;
 186         } else {
 187                 use_bank = use_ring = false;
 188         }
 189
 190         *pos &= (1UL << 22) - 1;
 191
 192         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 193         if (r < 0) {
 194                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 195                 return r;
 196         }
 197
 198         r = amdgpu_virt_enable_access_debugfs(adev);
 199         if (r < 0) {
 200                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 201                 return r;
 202         }
 203
 204         if (use_bank) {
 205                 if ((sh_bank != 0xFFFFFFFF && sh_bank >= adev->gfx.config.max_sh_per_se) ||
 206                     (se_bank != 0xFFFFFFFF && se_bank >= adev->gfx.config.max_shader_engines)) {
 207                         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 208                         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 209                         amdgpu_virt_disable_access_debugfs(adev);
 210                         return -EINVAL;
 211                 }
 212                 mutex_lock(&adev->grbm_idx_mutex);
 213                 amdgpu_gfx_select_se_sh(adev, se_bank,
 214                                         sh_bank, instance_bank);
 215         } else if (use_ring) {
 216                 mutex_lock(&adev->srbm_mutex);
 217                 amdgpu_gfx_select_me_pipe_q(adev, me, pipe, queue, vmid);
 218         }
 219
 220         if (pm_pg_lock)
 221                 mutex_lock(&adev->pm.mutex);
 222
 223         while (size) {
 224                 uint32_t value;
 225
 226                 if (read) {
 227                         value = RREG32(*pos >> 2);
 228                         r = put_user(value, (uint32_t *)buf);
 229                 } else {
 230                         r = get_user(value, (uint32_t *)buf);
 231                         if (!r)
 232                                 amdgpu_mm_wreg_mmio_rlc(adev, *pos >> 2, value);
 233                 }
 234                 if (r) {
 235                         result = r;
 236                         goto end;
 237                 }
 238
 239                 result += 4;
 240                 buf += 4;
 241                 *pos += 4;
 242                 size -= 4;
 243         }
 244
 245 end:
 246         if (use_bank) {
 247                 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
 248                 mutex_unlock(&adev->grbm_idx_mutex);
 249         } else if (use_ring) {
 250                 amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0);
 251                 mutex_unlock(&adev->srbm_mutex);
 252         }
 253
 254         if (pm_pg_lock)
 255                 mutex_unlock(&adev->pm.mutex);
 256
 257         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 258         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 259
 260         amdgpu_virt_disable_access_debugfs(adev);
 261         return result;
 262 }
 263
 264 /*
 265  * amdgpu_debugfs_regs_read - Callback for reading MMIO registers
 266  */
 267 static ssize_t amdgpu_debugfs_regs_read(struct file *f, char __user *buf,
 268                                         size_t size, loff_t *pos)
 269 {
 270         return amdgpu_debugfs_process_reg_op(true, f, buf, size, pos);
 271 }
 272
 273 /*
 274  * amdgpu_debugfs_regs_write - Callback for writing MMIO registers
 275  */
 276 static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,
 277                                          size_t size, loff_t *pos)
 278 {
 279         return amdgpu_debugfs_process_reg_op(false, f, (char __user *)buf, size, pos);
 280 }
 281
 282
 283 /**
 284  * amdgpu_debugfs_regs_pcie_read - Read from a PCIE register
 285  *
 286  * @f: open file handle
 287  * @buf: User buffer to store read data in
 288  * @size: Number of bytes to read
 289  * @pos:  Offset to seek to
 290  *
 291  * The lower bits are the BYTE offset of the register to read.  This
 292  * allows reading multiple registers in a single call and having
 293  * the returned size reflect that.
 294  */
 295 static ssize_t amdgpu_debugfs_regs_pcie_read(struct file *f, char __user *buf,
 296                                         size_t size, loff_t *pos)
 297 {
 298         struct amdgpu_device *adev = file_inode(f)->i_private;
 299         ssize_t result = 0;
 300         int r;
 301
 302         if (size & 0x3 || *pos & 0x3)
 303                 return -EINVAL;
 304
 305         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 306         if (r < 0) {
 307                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 308                 return r;
 309         }
 310
 311         r = amdgpu_virt_enable_access_debugfs(adev);
 312         if (r < 0) {
 313                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 314                 return r;
 315         }
 316
 317         while (size) {
 318                 uint32_t value;
 319
 320                 value = RREG32_PCIE(*pos);
 321                 r = put_user(value, (uint32_t *)buf);
 322                 if (r) {
 323                         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 324                         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 325                         amdgpu_virt_disable_access_debugfs(adev);
 326                         return r;
 327                 }
 328
 329                 result += 4;
 330                 buf += 4;
 331                 *pos += 4;
 332                 size -= 4;
 333         }
 334
 335         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 336         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 337
 338         amdgpu_virt_disable_access_debugfs(adev);
 339         return result;
 340 }
 341
 342 /**
 343  * amdgpu_debugfs_regs_pcie_write - Write to a PCIE register
 344  *
 345  * @f: open file handle
 346  * @buf: User buffer to write data from
 347  * @size: Number of bytes to write
 348  * @pos:  Offset to seek to
 349  *
 350  * The lower bits are the BYTE offset of the register to write.  This
 351  * allows writing multiple registers in a single call and having
 352  * the returned size reflect that.
 353  */
 354 static ssize_t amdgpu_debugfs_regs_pcie_write(struct file *f, const char __user *buf,
 355                                          size_t size, loff_t *pos)
 356 {
 357         struct amdgpu_device *adev = file_inode(f)->i_private;
 358         ssize_t result = 0;
 359         int r;
 360
 361         if (size & 0x3 || *pos & 0x3)
 362                 return -EINVAL;
 363
 364         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 365         if (r < 0) {
 366                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 367                 return r;
 368         }
 369
 370         r = amdgpu_virt_enable_access_debugfs(adev);
 371         if (r < 0) {
 372                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 373                 return r;
 374         }
 375
 376         while (size) {
 377                 uint32_t value;
 378
 379                 r = get_user(value, (uint32_t *)buf);
 380                 if (r) {
 381                         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 382                         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 383                         amdgpu_virt_disable_access_debugfs(adev);
 384                         return r;
 385                 }
 386
 387                 WREG32_PCIE(*pos, value);
 388
 389                 result += 4;
 390                 buf += 4;
 391                 *pos += 4;
 392                 size -= 4;
 393         }
 394
 395         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 396         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 397
 398         amdgpu_virt_disable_access_debugfs(adev);
 399         return result;
 400 }
 401
 402 /**
 403  * amdgpu_debugfs_regs_didt_read - Read from a DIDT register
 404  *
 405  * @f: open file handle
 406  * @buf: User buffer to store read data in
 407  * @size: Number of bytes to read
 408  * @pos:  Offset to seek to
 409  *
 410  * The lower bits are the BYTE offset of the register to read.  This
 411  * allows reading multiple registers in a single call and having
 412  * the returned size reflect that.
 413  */
 414 static ssize_t amdgpu_debugfs_regs_didt_read(struct file *f, char __user *buf,
 415                                         size_t size, loff_t *pos)
 416 {
 417         struct amdgpu_device *adev = file_inode(f)->i_private;
 418         ssize_t result = 0;
 419         int r;
 420
 421         if (size & 0x3 || *pos & 0x3)
 422                 return -EINVAL;
 423
 424         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 425         if (r < 0) {
 426                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 427                 return r;
 428         }
 429
 430         r = amdgpu_virt_enable_access_debugfs(adev);
 431         if (r < 0) {
 432                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 433                 return r;
 434         }
 435
 436         while (size) {
 437                 uint32_t value;
 438
 439                 value = RREG32_DIDT(*pos >> 2);
 440                 r = put_user(value, (uint32_t *)buf);
 441                 if (r) {
 442                         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 443                         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 444                         amdgpu_virt_disable_access_debugfs(adev);
 445                         return r;
 446                 }
 447
 448                 result += 4;
 449                 buf += 4;
 450                 *pos += 4;
 451                 size -= 4;
 452         }
 453
 454         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 455         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 456
 457         amdgpu_virt_disable_access_debugfs(adev);
 458         return result;
 459 }
 460
 461 /**
 462  * amdgpu_debugfs_regs_didt_write - Write to a DIDT register
 463  *
 464  * @f: open file handle
 465  * @buf: User buffer to write data from
 466  * @size: Number of bytes to write
 467  * @pos:  Offset to seek to
 468  *
 469  * The lower bits are the BYTE offset of the register to write.  This
 470  * allows writing multiple registers in a single call and having
 471  * the returned size reflect that.
 472  */
 473 static ssize_t amdgpu_debugfs_regs_didt_write(struct file *f, const char __user *buf,
 474                                          size_t size, loff_t *pos)
 475 {
 476         struct amdgpu_device *adev = file_inode(f)->i_private;
 477         ssize_t result = 0;
 478         int r;
 479
 480         if (size & 0x3 || *pos & 0x3)
 481                 return -EINVAL;
 482
 483         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 484         if (r < 0) {
 485                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 486                 return r;
 487         }
 488
 489         r = amdgpu_virt_enable_access_debugfs(adev);
 490         if (r < 0) {
 491                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 492                 return r;
 493         }
 494
 495         while (size) {
 496                 uint32_t value;
 497
 498                 r = get_user(value, (uint32_t *)buf);
 499                 if (r) {
 500                         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 501                         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 502                         amdgpu_virt_disable_access_debugfs(adev);
 503                         return r;
 504                 }
 505
 506                 WREG32_DIDT(*pos >> 2, value);
 507
 508                 result += 4;
 509                 buf += 4;
 510                 *pos += 4;
 511                 size -= 4;
 512         }
 513
 514         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 515         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 516
 517         amdgpu_virt_disable_access_debugfs(adev);
 518         return result;
 519 }
 520
 521 /**
 522  * amdgpu_debugfs_regs_smc_read - Read from a SMC register
 523  *
 524  * @f: open file handle
 525  * @buf: User buffer to store read data in
 526  * @size: Number of bytes to read
 527  * @pos:  Offset to seek to
 528  *
 529  * The lower bits are the BYTE offset of the register to read.  This
 530  * allows reading multiple registers in a single call and having
 531  * the returned size reflect that.
 532  */
 533 static ssize_t amdgpu_debugfs_regs_smc_read(struct file *f, char __user *buf,
 534                                         size_t size, loff_t *pos)
 535 {
 536         struct amdgpu_device *adev = file_inode(f)->i_private;
 537         ssize_t result = 0;
 538         int r;
 539
 540         if (size & 0x3 || *pos & 0x3)
 541                 return -EINVAL;
 542
 543         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 544         if (r < 0) {
 545                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 546                 return r;
 547         }
 548
 549         r = amdgpu_virt_enable_access_debugfs(adev);
 550         if (r < 0) {
 551                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 552                 return r;
 553         }
 554
 555         while (size) {
 556                 uint32_t value;
 557
 558                 value = RREG32_SMC(*pos);
 559                 r = put_user(value, (uint32_t *)buf);
 560                 if (r) {
 561                         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 562                         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 563                         amdgpu_virt_disable_access_debugfs(adev);
 564                         return r;
 565                 }
 566
 567                 result += 4;
 568                 buf += 4;
 569                 *pos += 4;
 570                 size -= 4;
 571         }
 572
 573         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 574         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 575
 576         amdgpu_virt_disable_access_debugfs(adev);
 577         return result;
 578 }
 579
 580 /**
 581  * amdgpu_debugfs_regs_smc_write - Write to a SMC register
 582  *
 583  * @f: open file handle
 584  * @buf: User buffer to write data from
 585  * @size: Number of bytes to write
 586  * @pos:  Offset to seek to
 587  *
 588  * The lower bits are the BYTE offset of the register to write.  This
 589  * allows writing multiple registers in a single call and having
 590  * the returned size reflect that.
 591  */
 592 static ssize_t amdgpu_debugfs_regs_smc_write(struct file *f, const char __user *buf,
 593                                          size_t size, loff_t *pos)
 594 {
 595         struct amdgpu_device *adev = file_inode(f)->i_private;
 596         ssize_t result = 0;
 597         int r;
 598
 599         if (size & 0x3 || *pos & 0x3)
 600                 return -EINVAL;
 601
 602         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 603         if (r < 0) {
 604                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 605                 return r;
 606         }
 607
 608         r = amdgpu_virt_enable_access_debugfs(adev);
 609         if (r < 0) {
 610                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 611                 return r;
 612         }
 613
 614         while (size) {
 615                 uint32_t value;
 616
 617                 r = get_user(value, (uint32_t *)buf);
 618                 if (r) {
 619                         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 620                         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 621                         amdgpu_virt_disable_access_debugfs(adev);
 622                         return r;
 623                 }
 624
 625                 WREG32_SMC(*pos, value);
 626
 627                 result += 4;
 628                 buf += 4;
 629                 *pos += 4;
 630                 size -= 4;
 631         }
 632
 633         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 634         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 635
 636         amdgpu_virt_disable_access_debugfs(adev);
 637         return result;
 638 }
 639
 640 /**
 641  * amdgpu_debugfs_gca_config_read - Read from gfx config data
 642  *
 643  * @f: open file handle
 644  * @buf: User buffer to store read data in
 645  * @size: Number of bytes to read
 646  * @pos:  Offset to seek to
 647  *
 648  * This file is used to access configuration data in a somewhat
 649  * stable fashion.  The format is a series of DWORDs with the first
 650  * indicating which revision it is.  New content is appended to the
 651  * end so that older software can still read the data.
 652  */
 653
 654 static ssize_t amdgpu_debugfs_gca_config_read(struct file *f, char __user *buf,
 655                                         size_t size, loff_t *pos)
 656 {
 657         struct amdgpu_device *adev = file_inode(f)->i_private;
 658         ssize_t result = 0;
 659         int r;
 660         uint32_t *config, no_regs = 0;
 661
 662         if (size & 0x3 || *pos & 0x3)
 663                 return -EINVAL;
 664
 665         config = kmalloc_array(256, sizeof(*config), GFP_KERNEL);
 666         if (!config)
 667                 return -ENOMEM;
 668
 669         /* version, increment each time something is added */
 670         config[no_regs++] = 3;
 671         config[no_regs++] = adev->gfx.config.max_shader_engines;
 672         config[no_regs++] = adev->gfx.config.max_tile_pipes;
 673         config[no_regs++] = adev->gfx.config.max_cu_per_sh;
 674         config[no_regs++] = adev->gfx.config.max_sh_per_se;
 675         config[no_regs++] = adev->gfx.config.max_backends_per_se;
 676         config[no_regs++] = adev->gfx.config.max_texture_channel_caches;
 677         config[no_regs++] = adev->gfx.config.max_gprs;
 678         config[no_regs++] = adev->gfx.config.max_gs_threads;
 679         config[no_regs++] = adev->gfx.config.max_hw_contexts;
 680         config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_frontend;
 681         config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_backend;
 682         config[no_regs++] = adev->gfx.config.sc_hiz_tile_fifo_size;
 683         config[no_regs++] = adev->gfx.config.sc_earlyz_tile_fifo_size;
 684         config[no_regs++] = adev->gfx.config.num_tile_pipes;
 685         config[no_regs++] = adev->gfx.config.backend_enable_mask;
 686         config[no_regs++] = adev->gfx.config.mem_max_burst_length_bytes;
 687         config[no_regs++] = adev->gfx.config.mem_row_size_in_kb;
 688         config[no_regs++] = adev->gfx.config.shader_engine_tile_size;
 689         config[no_regs++] = adev->gfx.config.num_gpus;
 690         config[no_regs++] = adev->gfx.config.multi_gpu_tile_size;
 691         config[no_regs++] = adev->gfx.config.mc_arb_ramcfg;
 692         config[no_regs++] = adev->gfx.config.gb_addr_config;
 693         config[no_regs++] = adev->gfx.config.num_rbs;
 694
 695         /* rev==1 */
 696         config[no_regs++] = adev->rev_id;
 697         config[no_regs++] = adev->pg_flags;
 698         config[no_regs++] = adev->cg_flags;
 699
 700         /* rev==2 */
 701         config[no_regs++] = adev->family;
 702         config[no_regs++] = adev->external_rev_id;
 703
 704         /* rev==3 */
 705         config[no_regs++] = adev->pdev->device;
 706         config[no_regs++] = adev->pdev->revision;
 707         config[no_regs++] = adev->pdev->subsystem_device;
 708         config[no_regs++] = adev->pdev->subsystem_vendor;
 709
 710         while (size && (*pos < no_regs * 4)) {
 711                 uint32_t value;
 712
 713                 value = config[*pos >> 2];
 714                 r = put_user(value, (uint32_t *)buf);
 715                 if (r) {
 716                         kfree(config);
 717                         return r;
 718                 }
 719
 720                 result += 4;
 721                 buf += 4;
 722                 *pos += 4;
 723                 size -= 4;
 724         }
 725
 726         kfree(config);
 727         return result;
 728 }
 729
 730 /**
 731  * amdgpu_debugfs_sensor_read - Read from the powerplay sensors
 732  *
 733  * @f: open file handle
 734  * @buf: User buffer to store read data in
 735  * @size: Number of bytes to read
 736  * @pos:  Offset to seek to
 737  *
 738  * The offset is treated as the BYTE address of one of the sensors
 739  * enumerated in amd/include/kgd_pp_interface.h under the
 740  * 'amd_pp_sensors' enumeration.  For instance to read the UVD VCLK
 741  * you would use the offset 3 * 4 = 12.
 742  */
 743 static ssize_t amdgpu_debugfs_sensor_read(struct file *f, char __user *buf,
 744                                         size_t size, loff_t *pos)
 745 {
 746         struct amdgpu_device *adev = file_inode(f)->i_private;
 747         int idx, x, outsize, r, valuesize;
 748         uint32_t values[16];
 749
 750         if (size & 3 || *pos & 0x3)
 751                 return -EINVAL;
 752
 753         if (!adev->pm.dpm_enabled)
 754                 return -EINVAL;
 755
 756         /* convert offset to sensor number */
 757         idx = *pos >> 2;
 758
 759         valuesize = sizeof(values);
 760
 761         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 762         if (r < 0) {
 763                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 764                 return r;
 765         }
 766
 767         r = amdgpu_virt_enable_access_debugfs(adev);
 768         if (r < 0) {
 769                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 770                 return r;
 771         }
 772
 773         r = amdgpu_dpm_read_sensor(adev, idx, &values[0], &valuesize);
 774
 775         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 776         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 777
 778         if (r) {
 779                 amdgpu_virt_disable_access_debugfs(adev);
 780                 return r;
 781         }
 782
 783         if (size > valuesize) {
 784                 amdgpu_virt_disable_access_debugfs(adev);
 785                 return -EINVAL;
 786         }
 787
 788         outsize = 0;
 789         x = 0;
 790         if (!r) {
 791                 while (size) {
 792                         r = put_user(values[x++], (int32_t *)buf);
 793                         buf += 4;
 794                         size -= 4;
 795                         outsize += 4;
 796                 }
 797         }
 798
 799         amdgpu_virt_disable_access_debugfs(adev);
 800         return !r ? outsize : r;
 801 }
 802
 803 /** amdgpu_debugfs_wave_read - Read WAVE STATUS data
 804  *
 805  * @f: open file handle
 806  * @buf: User buffer to store read data in
 807  * @size: Number of bytes to read
 808  * @pos:  Offset to seek to
 809  *
 810  * The offset being sought changes which wave that the status data
 811  * will be returned for.  The bits are used as follows:
 812  *
 813  * Bits 0..6:   Byte offset into data
 814  * Bits 7..14:  SE selector
 815  * Bits 15..22: SH/SA selector
 816  * Bits 23..30: CU/{WGP+SIMD} selector
 817  * Bits 31..36: WAVE ID selector
 818  * Bits 37..44: SIMD ID selector
 819  *
 820  * The returned data begins with one DWORD of version information
 821  * Followed by WAVE STATUS registers relevant to the GFX IP version
 822  * being used.  See gfx_v8_0_read_wave_data() for an example output.
 823  */
 824 static ssize_t amdgpu_debugfs_wave_read(struct file *f, char __user *buf,
 825                                         size_t size, loff_t *pos)
 826 {
 827         struct amdgpu_device *adev = f->f_inode->i_private;
 828         int r, x;
 829         ssize_t result = 0;
 830         uint32_t offset, se, sh, cu, wave, simd, data[32];
 831
 832         if (size & 3 || *pos & 3)
 833                 return -EINVAL;
 834
 835         /* decode offset */
 836         offset = (*pos & GENMASK_ULL(6, 0));
 837         se = (*pos & GENMASK_ULL(14, 7)) >> 7;
 838         sh = (*pos & GENMASK_ULL(22, 15)) >> 15;
 839         cu = (*pos & GENMASK_ULL(30, 23)) >> 23;
 840         wave = (*pos & GENMASK_ULL(36, 31)) >> 31;
 841         simd = (*pos & GENMASK_ULL(44, 37)) >> 37;
 842
 843         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 844         if (r < 0) {
 845                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 846                 return r;
 847         }
 848
 849         r = amdgpu_virt_enable_access_debugfs(adev);
 850         if (r < 0) {
 851                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 852                 return r;
 853         }
 854
 855         /* switch to the specific se/sh/cu */
 856         mutex_lock(&adev->grbm_idx_mutex);
 857         amdgpu_gfx_select_se_sh(adev, se, sh, cu);
 858
 859         x = 0;
 860         if (adev->gfx.funcs->read_wave_data)
 861                 adev->gfx.funcs->read_wave_data(adev, simd, wave, data, &x);
 862
 863         amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF);
 864         mutex_unlock(&adev->grbm_idx_mutex);
 865
 866         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 867         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 868
 869         if (!x) {
 870                 amdgpu_virt_disable_access_debugfs(adev);
 871                 return -EINVAL;
 872         }
 873
 874         while (size && (offset < x * 4)) {
 875                 uint32_t value;
 876
 877                 value = data[offset >> 2];
 878                 r = put_user(value, (uint32_t *)buf);
 879                 if (r) {
 880                         amdgpu_virt_disable_access_debugfs(adev);
 881                         return r;
 882                 }
 883
 884                 result += 4;
 885                 buf += 4;
 886                 offset += 4;
 887                 size -= 4;
 888         }
 889
 890         amdgpu_virt_disable_access_debugfs(adev);
 891         return result;
 892 }
 893
 894 /** amdgpu_debugfs_gpr_read - Read wave gprs
 895  *
 896  * @f: open file handle
 897  * @buf: User buffer to store read data in
 898  * @size: Number of bytes to read
 899  * @pos:  Offset to seek to
 900  *
 901  * The offset being sought changes which wave that the status data
 902  * will be returned for.  The bits are used as follows:
 903  *
 904  * Bits 0..11:  Byte offset into data
 905  * Bits 12..19: SE selector
 906  * Bits 20..27: SH/SA selector
 907  * Bits 28..35: CU/{WGP+SIMD} selector
 908  * Bits 36..43: WAVE ID selector
 909  * Bits 37..44: SIMD ID selector
 910  * Bits 52..59: Thread selector
 911  * Bits 60..61: Bank selector (VGPR=0,SGPR=1)
 912  *
 913  * The return data comes from the SGPR or VGPR register bank for
 914  * the selected operational unit.
 915  */
 916 static ssize_t amdgpu_debugfs_gpr_read(struct file *f, char __user *buf,
 917                                         size_t size, loff_t *pos)
 918 {
 919         struct amdgpu_device *adev = f->f_inode->i_private;
 920         int r;
 921         ssize_t result = 0;
 922         uint32_t offset, se, sh, cu, wave, simd, thread, bank, *data;
 923
 924         if (size > 4096 || size & 3 || *pos & 3)
 925                 return -EINVAL;
 926
 927         /* decode offset */
 928         offset = (*pos & GENMASK_ULL(11, 0)) >> 2;
 929         se = (*pos & GENMASK_ULL(19, 12)) >> 12;
 930         sh = (*pos & GENMASK_ULL(27, 20)) >> 20;
 931         cu = (*pos & GENMASK_ULL(35, 28)) >> 28;
 932         wave = (*pos & GENMASK_ULL(43, 36)) >> 36;
 933         simd = (*pos & GENMASK_ULL(51, 44)) >> 44;
 934         thread = (*pos & GENMASK_ULL(59, 52)) >> 52;
 935         bank = (*pos & GENMASK_ULL(61, 60)) >> 60;
 936
 937         data = kcalloc(1024, sizeof(*data), GFP_KERNEL);
 938         if (!data)
 939                 return -ENOMEM;
 940
 941         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 942         if (r < 0)
 943                 goto err;
 944
 945         r = amdgpu_virt_enable_access_debugfs(adev);
 946         if (r < 0)
 947                 goto err;
 948
 949         /* switch to the specific se/sh/cu */
 950         mutex_lock(&adev->grbm_idx_mutex);
 951         amdgpu_gfx_select_se_sh(adev, se, sh, cu);
 952
 953         if (bank == 0) {
 954                 if (adev->gfx.funcs->read_wave_vgprs)
 955                         adev->gfx.funcs->read_wave_vgprs(adev, simd, wave, thread, offset, size>>2, data);
 956         } else {
 957                 if (adev->gfx.funcs->read_wave_sgprs)
 958                         adev->gfx.funcs->read_wave_sgprs(adev, simd, wave, offset, size>>2, data);
 959         }
 960
 961         amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF);
 962         mutex_unlock(&adev->grbm_idx_mutex);
 963
 964         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 965         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 966
 967         while (size) {
 968                 uint32_t value;
 969
 970                 value = data[result >> 2];
 971                 r = put_user(value, (uint32_t *)buf);
 972                 if (r) {
 973                         amdgpu_virt_disable_access_debugfs(adev);
 974                         goto err;
 975                 }
 976
 977                 result += 4;
 978                 buf += 4;
 979                 size -= 4;
 980         }
 981
 982         kfree(data);
 983         amdgpu_virt_disable_access_debugfs(adev);
 984         return result;
 985
 986 err:
 987         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 988         kfree(data);
 989         return r;
 990 }
 991
 992 /**
 993  * amdgpu_debugfs_regs_gfxoff_write - Enable/disable GFXOFF
 994  *
 995  * @f: open file handle
 996  * @buf: User buffer to write data from
 997  * @size: Number of bytes to write
 998  * @pos:  Offset to seek to
 999  *
1000  * Write a 32-bit zero to disable or a 32-bit non-zero to enable
1001  */
1002 static ssize_t amdgpu_debugfs_gfxoff_write(struct file *f, const char __user *buf,
1003                                          size_t size, loff_t *pos)
1004 {
1005         struct amdgpu_device *adev = file_inode(f)->i_private;
1006         ssize_t result = 0;
1007         int r;
1008
1009         if (size & 0x3 || *pos & 0x3)
1010                 return -EINVAL;
1011
1012         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1013         if (r < 0) {
1014                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1015                 return r;
1016         }
1017
1018         while (size) {
1019                 uint32_t value;
1020
1021                 r = get_user(value, (uint32_t *)buf);
1022                 if (r) {
1023                         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1024                         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1025                         return r;
1026                 }
1027
1028                 amdgpu_gfx_off_ctrl(adev, value ? true : false);
1029
1030                 result += 4;
1031                 buf += 4;
1032                 *pos += 4;
1033                 size -= 4;
1034         }
1035
1036         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1037         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1038
1039         return result;
1040 }
1041
1042
1043 /**
1044  * amdgpu_debugfs_regs_gfxoff_status - read gfxoff status
1045  *
1046  * @f: open file handle
1047  * @buf: User buffer to store read data in
1048  * @size: Number of bytes to read
1049  * @pos:  Offset to seek to
1050  */
1051 static ssize_t amdgpu_debugfs_gfxoff_read(struct file *f, char __user *buf,
1052                                          size_t size, loff_t *pos)
1053 {
1054         struct amdgpu_device *adev = file_inode(f)->i_private;
1055         ssize_t result = 0;
1056         int r;
1057
1058         if (size & 0x3 || *pos & 0x3)
1059                 return -EINVAL;
1060
1061         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1062         if (r < 0)
1063                 return r;
1064
1065         while (size) {
1066                 uint32_t value;
1067
1068                 r = amdgpu_get_gfx_off_status(adev, &value);
1069                 if (r) {
1070                         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1071                         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1072                         return r;
1073                 }
1074
1075                 r = put_user(value, (uint32_t *)buf);
1076                 if (r) {
1077                         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1078                         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1079                         return r;
1080                 }
1081
1082                 result += 4;
1083                 buf += 4;
1084                 *pos += 4;
1085                 size -= 4;
1086         }
1087
1088         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1089         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1090
1091         return result;
1092 }
1093
1094 static const struct file_operations amdgpu_debugfs_regs_fops = {
1095         .owner = THIS_MODULE,
1096         .read = amdgpu_debugfs_regs_read,
1097         .write = amdgpu_debugfs_regs_write,
1098         .llseek = default_llseek
1099 };
1100 static const struct file_operations amdgpu_debugfs_regs_didt_fops = {
1101         .owner = THIS_MODULE,
1102         .read = amdgpu_debugfs_regs_didt_read,
1103         .write = amdgpu_debugfs_regs_didt_write,
1104         .llseek = default_llseek
1105 };
1106 static const struct file_operations amdgpu_debugfs_regs_pcie_fops = {
1107         .owner = THIS_MODULE,
1108         .read = amdgpu_debugfs_regs_pcie_read,
1109         .write = amdgpu_debugfs_regs_pcie_write,
1110         .llseek = default_llseek
1111 };
1112 static const struct file_operations amdgpu_debugfs_regs_smc_fops = {
1113         .owner = THIS_MODULE,
1114         .read = amdgpu_debugfs_regs_smc_read,
1115         .write = amdgpu_debugfs_regs_smc_write,
1116         .llseek = default_llseek
1117 };
1118
1119 static const struct file_operations amdgpu_debugfs_gca_config_fops = {
1120         .owner = THIS_MODULE,
1121         .read = amdgpu_debugfs_gca_config_read,
1122         .llseek = default_llseek
1123 };
1124
1125 static const struct file_operations amdgpu_debugfs_sensors_fops = {
1126         .owner = THIS_MODULE,
1127         .read = amdgpu_debugfs_sensor_read,
1128         .llseek = default_llseek
1129 };
1130
1131 static const struct file_operations amdgpu_debugfs_wave_fops = {
1132         .owner = THIS_MODULE,
1133         .read = amdgpu_debugfs_wave_read,
1134         .llseek = default_llseek
1135 };
1136 static const struct file_operations amdgpu_debugfs_gpr_fops = {
1137         .owner = THIS_MODULE,
1138         .read = amdgpu_debugfs_gpr_read,
1139         .llseek = default_llseek
1140 };
1141
1142 static const struct file_operations amdgpu_debugfs_gfxoff_fops = {
1143         .owner = THIS_MODULE,
1144         .read = amdgpu_debugfs_gfxoff_read,
1145         .write = amdgpu_debugfs_gfxoff_write,
1146         .llseek = default_llseek
1147 };
1148
1149 static const struct file_operations *debugfs_regs[] = {
1150         &amdgpu_debugfs_regs_fops,
1151         &amdgpu_debugfs_regs_didt_fops,
1152         &amdgpu_debugfs_regs_pcie_fops,
1153         &amdgpu_debugfs_regs_smc_fops,
1154         &amdgpu_debugfs_gca_config_fops,
1155         &amdgpu_debugfs_sensors_fops,
1156         &amdgpu_debugfs_wave_fops,
1157         &amdgpu_debugfs_gpr_fops,
1158         &amdgpu_debugfs_gfxoff_fops,
1159 };
1160
1161 static const char *debugfs_regs_names[] = {
1162         "amdgpu_regs",
1163         "amdgpu_regs_didt",
1164         "amdgpu_regs_pcie",
1165         "amdgpu_regs_smc",
1166         "amdgpu_gca_config",
1167         "amdgpu_sensors",
1168         "amdgpu_wave",
1169         "amdgpu_gpr",
1170         "amdgpu_gfxoff",
1171 };
1172
1173 /**
1174  * amdgpu_debugfs_regs_init -   Initialize debugfs entries that provide
1175  *                              register access.
1176  *
1177  * @adev: The device to attach the debugfs entries to
1178  */
1179 int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
1180 {
1181         struct drm_minor *minor = adev_to_drm(adev)->primary;
1182         struct dentry *ent, *root = minor->debugfs_root;
1183         unsigned int i;
1184
1185         for (i = 0; i < ARRAY_SIZE(debugfs_regs); i++) {
1186                 ent = debugfs_create_file(debugfs_regs_names[i],
1187                                           S_IFREG | S_IRUGO, root,
1188                                           adev, debugfs_regs[i]);
1189                 if (!i && !IS_ERR_OR_NULL(ent))
1190                         i_size_write(ent->d_inode, adev->rmmio_size);
1191         }
1192
1193         return 0;
1194 }
1195
1196 static int amdgpu_debugfs_test_ib_show(struct seq_file *m, void *unused)
1197 {
1198         struct amdgpu_device *adev = (struct amdgpu_device *)m->private;
1199         struct drm_device *dev = adev_to_drm(adev);
1200         int r = 0, i;
1201
1202         r = pm_runtime_get_sync(dev->dev);
1203         if (r < 0) {
1204                 pm_runtime_put_autosuspend(dev->dev);
1205                 return r;
1206         }
1207
1208         /* Avoid accidently unparking the sched thread during GPU reset */
1209         r = down_read_killable(&adev->reset_sem);
1210         if (r)
1211                 return r;
1212
1213         /* hold on the scheduler */
1214         for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
1215                 struct amdgpu_ring *ring = adev->rings[i];
1216
1217                 if (!ring || !ring->sched.thread)
1218                         continue;
1219                 kthread_park(ring->sched.thread);
1220         }
1221
1222         seq_printf(m, "run ib test:\n");
1223         r = amdgpu_ib_ring_tests(adev);
1224         if (r)
1225                 seq_printf(m, "ib ring tests failed (%d).\n", r);
1226         else
1227                 seq_printf(m, "ib ring tests passed.\n");
1228
1229         /* go on the scheduler */
1230         for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
1231                 struct amdgpu_ring *ring = adev->rings[i];
1232
1233                 if (!ring || !ring->sched.thread)
1234                         continue;
1235                 kthread_unpark(ring->sched.thread);
1236         }
1237
1238         up_read(&adev->reset_sem);
1239
1240         pm_runtime_mark_last_busy(dev->dev);
1241         pm_runtime_put_autosuspend(dev->dev);
1242
1243         return 0;
1244 }
1245
1246 static int amdgpu_debugfs_evict_vram(void *data, u64 *val)
1247 {
1248         struct amdgpu_device *adev = (struct amdgpu_device *)data;
1249         struct drm_device *dev = adev_to_drm(adev);
1250         int r;
1251
1252         r = pm_runtime_get_sync(dev->dev);
1253         if (r < 0) {
1254                 pm_runtime_put_autosuspend(dev->dev);
1255                 return r;
1256         }
1257
1258         *val = amdgpu_bo_evict_vram(adev);
1259
1260         pm_runtime_mark_last_busy(dev->dev);
1261         pm_runtime_put_autosuspend(dev->dev);
1262
1263         return 0;
1264 }
1265
1266
1267 static int amdgpu_debugfs_evict_gtt(void *data, u64 *val)
1268 {
1269         struct amdgpu_device *adev = (struct amdgpu_device *)data;
1270         struct drm_device *dev = adev_to_drm(adev);
1271         struct ttm_resource_manager *man;
1272         int r;
1273
1274         r = pm_runtime_get_sync(dev->dev);
1275         if (r < 0) {
1276                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1277                 return r;
1278         }
1279
1280         man = ttm_manager_type(&adev->mman.bdev, TTM_PL_TT);
1281         *val = ttm_resource_manager_evict_all(&adev->mman.bdev, man);
1282
1283         pm_runtime_mark_last_busy(dev->dev);
1284         pm_runtime_put_autosuspend(dev->dev);
1285
1286         return 0;
1287 }
1288
1289
1290 static int amdgpu_debugfs_vm_info_show(struct seq_file *m, void *unused)
1291 {
1292         struct amdgpu_device *adev = (struct amdgpu_device *)m->private;
1293         struct drm_device *dev = adev_to_drm(adev);
1294         struct drm_file *file;
1295         int r;
1296
1297         r = mutex_lock_interruptible(&dev->filelist_mutex);
1298         if (r)
1299                 return r;
1300
1301         list_for_each_entry(file, &dev->filelist, lhead) {
1302                 struct amdgpu_fpriv *fpriv = file->driver_priv;
1303                 struct amdgpu_vm *vm = &fpriv->vm;
1304
1305                 seq_printf(m, "pid:%d\tProcess:%s ----------\n",
1306                                 vm->task_info.pid, vm->task_info.process_name);
1307                 r = amdgpu_bo_reserve(vm->root.base.bo, true);
1308                 if (r)
1309                         break;
1310                 amdgpu_debugfs_vm_bo_info(vm, m);
1311                 amdgpu_bo_unreserve(vm->root.base.bo);
1312         }
1313
1314         mutex_unlock(&dev->filelist_mutex);
1315
1316         return r;
1317 }
1318
1319 DEFINE_SHOW_ATTRIBUTE(amdgpu_debugfs_test_ib);
1320 DEFINE_SHOW_ATTRIBUTE(amdgpu_debugfs_vm_info);
1321 DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_evict_vram_fops, amdgpu_debugfs_evict_vram,
1322                          NULL, "%lld\n");
1323 DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_evict_gtt_fops, amdgpu_debugfs_evict_gtt,
1324                          NULL, "%lld\n");
1325
1326 static void amdgpu_ib_preempt_fences_swap(struct amdgpu_ring *ring,
1327                                           struct dma_fence **fences)
1328 {
1329         struct amdgpu_fence_driver *drv = &ring->fence_drv;
1330         uint32_t sync_seq, last_seq;
1331
1332         last_seq = atomic_read(&ring->fence_drv.last_seq);
1333         sync_seq = ring->fence_drv.sync_seq;
1334
1335         last_seq &= drv->num_fences_mask;
1336         sync_seq &= drv->num_fences_mask;
1337
1338         do {
1339                 struct dma_fence *fence, **ptr;
1340
1341                 ++last_seq;
1342                 last_seq &= drv->num_fences_mask;
1343                 ptr = &drv->fences[last_seq];
1344
1345                 fence = rcu_dereference_protected(*ptr, 1);
1346                 RCU_INIT_POINTER(*ptr, NULL);
1347
1348                 if (!fence)
1349                         continue;
1350
1351                 fences[last_seq] = fence;
1352
1353         } while (last_seq != sync_seq);
1354 }
1355
1356 static void amdgpu_ib_preempt_signal_fences(struct dma_fence **fences,
1357                                             int length)
1358 {
1359         int i;
1360         struct dma_fence *fence;
1361
1362         for (i = 0; i < length; i++) {
1363                 fence = fences[i];
1364                 if (!fence)
1365                         continue;
1366                 dma_fence_signal(fence);
1367                 dma_fence_put(fence);
1368         }
1369 }
1370
1371 static void amdgpu_ib_preempt_job_recovery(struct drm_gpu_scheduler *sched)
1372 {
1373         struct drm_sched_job *s_job;
1374         struct dma_fence *fence;
1375
1376         spin_lock(&sched->job_list_lock);
1377         list_for_each_entry(s_job, &sched->pending_list, list) {
1378                 fence = sched->ops->run_job(s_job);
1379                 dma_fence_put(fence);
1380         }
1381         spin_unlock(&sched->job_list_lock);
1382 }
1383
1384 static void amdgpu_ib_preempt_mark_partial_job(struct amdgpu_ring *ring)
1385 {
1386         struct amdgpu_job *job;
1387         struct drm_sched_job *s_job, *tmp;
1388         uint32_t preempt_seq;
1389         struct dma_fence *fence, **ptr;
1390         struct amdgpu_fence_driver *drv = &ring->fence_drv;
1391         struct drm_gpu_scheduler *sched = &ring->sched;
1392         bool preempted = true;
1393
1394         if (ring->funcs->type != AMDGPU_RING_TYPE_GFX)
1395                 return;
1396
1397         preempt_seq = le32_to_cpu(*(drv->cpu_addr + 2));
1398         if (preempt_seq <= atomic_read(&drv->last_seq)) {
1399                 preempted = false;
1400                 goto no_preempt;
1401         }
1402
1403         preempt_seq &= drv->num_fences_mask;
1404         ptr = &drv->fences[preempt_seq];
1405         fence = rcu_dereference_protected(*ptr, 1);
1406
1407 no_preempt:
1408         spin_lock(&sched->job_list_lock);
1409         list_for_each_entry_safe(s_job, tmp, &sched->pending_list, list) {
1410                 if (dma_fence_is_signaled(&s_job->s_fence->finished)) {
1411                         /* remove job from ring_mirror_list */
1412                         list_del_init(&s_job->list);
1413                         sched->ops->free_job(s_job);
1414                         continue;
1415                 }
1416                 job = to_amdgpu_job(s_job);
1417                 if (preempted && job->fence == fence)
1418                         /* mark the job as preempted */
1419                         job->preemption_status |= AMDGPU_IB_PREEMPTED;
1420         }
1421         spin_unlock(&sched->job_list_lock);
1422 }
1423
1424 static int amdgpu_debugfs_ib_preempt(void *data, u64 val)
1425 {
1426         int r, resched, length;
1427         struct amdgpu_ring *ring;
1428         struct dma_fence **fences = NULL;
1429         struct amdgpu_device *adev = (struct amdgpu_device *)data;
1430
1431         if (val >= AMDGPU_MAX_RINGS)
1432                 return -EINVAL;
1433
1434         ring = adev->rings[val];
1435
1436         if (!ring || !ring->funcs->preempt_ib || !ring->sched.thread)
1437                 return -EINVAL;
1438
1439         /* the last preemption failed */
1440         if (ring->trail_seq != le32_to_cpu(*ring->trail_fence_cpu_addr))
1441                 return -EBUSY;
1442
1443         length = ring->fence_drv.num_fences_mask + 1;
1444         fences = kcalloc(length, sizeof(void *), GFP_KERNEL);
1445         if (!fences)
1446                 return -ENOMEM;
1447
1448         /* Avoid accidently unparking the sched thread during GPU reset */
1449         r = down_read_killable(&adev->reset_sem);
1450         if (r)
1451                 goto pro_end;
1452
1453         /* stop the scheduler */
1454         kthread_park(ring->sched.thread);
1455
1456         resched = ttm_bo_lock_delayed_workqueue(&adev->mman.bdev);
1457
1458         /* preempt the IB */
1459         r = amdgpu_ring_preempt_ib(ring);
1460         if (r) {
1461                 DRM_WARN("failed to preempt ring %d\n", ring->idx);
1462                 goto failure;
1463         }
1464
1465         amdgpu_fence_process(ring);
1466
1467         if (atomic_read(&ring->fence_drv.last_seq) !=
1468             ring->fence_drv.sync_seq) {
1469                 DRM_INFO("ring %d was preempted\n", ring->idx);
1470
1471                 amdgpu_ib_preempt_mark_partial_job(ring);
1472
1473                 /* swap out the old fences */
1474                 amdgpu_ib_preempt_fences_swap(ring, fences);
1475
1476                 amdgpu_fence_driver_force_completion(ring);
1477
1478                 /* resubmit unfinished jobs */
1479                 amdgpu_ib_preempt_job_recovery(&ring->sched);
1480
1481                 /* wait for jobs finished */
1482                 amdgpu_fence_wait_empty(ring);
1483
1484                 /* signal the old fences */
1485                 amdgpu_ib_preempt_signal_fences(fences, length);
1486         }
1487
1488 failure:
1489         /* restart the scheduler */
1490         kthread_unpark(ring->sched.thread);
1491
1492         up_read(&adev->reset_sem);
1493
1494         ttm_bo_unlock_delayed_workqueue(&adev->mman.bdev, resched);
1495
1496 pro_end:
1497         kfree(fences);
1498
1499         return r;
1500 }
1501
1502 static int amdgpu_debugfs_sclk_set(void *data, u64 val)
1503 {
1504         int ret = 0;
1505         uint32_t max_freq, min_freq;
1506         struct amdgpu_device *adev = (struct amdgpu_device *)data;
1507
1508         if (amdgpu_sriov_vf(adev) && !amdgpu_sriov_is_pp_one_vf(adev))
1509                 return -EINVAL;
1510
1511         ret = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1512         if (ret < 0) {
1513                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1514                 return ret;
1515         }
1516
1517         if (is_support_sw_smu(adev)) {
1518                 ret = smu_get_dpm_freq_range(&adev->smu, SMU_SCLK, &min_freq, &max_freq);
1519                 if (ret || val > max_freq || val < min_freq)
1520                         return -EINVAL;
1521                 ret = smu_set_soft_freq_range(&adev->smu, SMU_SCLK, (uint32_t)val, (uint32_t)val);
1522         } else {
1523                 return 0;
1524         }
1525
1526         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1527         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1528
1529         if (ret)
1530                 return -EINVAL;
1531
1532         return 0;
1533 }
1534
1535 DEFINE_DEBUGFS_ATTRIBUTE(fops_ib_preempt, NULL,
1536                         amdgpu_debugfs_ib_preempt, "%llu\n");
1537
1538 DEFINE_DEBUGFS_ATTRIBUTE(fops_sclk_set, NULL,
1539                         amdgpu_debugfs_sclk_set, "%llu\n");
1540
1541 int amdgpu_debugfs_init(struct amdgpu_device *adev)
1542 {
1543         struct dentry *root = adev_to_drm(adev)->primary->debugfs_root;
1544         struct dentry *ent;
1545         int r, i;
1546
1547
1548
1549         ent = debugfs_create_file("amdgpu_preempt_ib", 0600, root, adev,
1550                                   &fops_ib_preempt);
1551         if (!ent) {
1552                 DRM_ERROR("unable to create amdgpu_preempt_ib debugsfs file\n");
1553                 return -EIO;
1554         }
1555
1556         ent = debugfs_create_file("amdgpu_force_sclk", 0200, root, adev,
1557                                   &fops_sclk_set);
1558         if (!ent) {
1559                 DRM_ERROR("unable to create amdgpu_set_sclk debugsfs file\n");
1560                 return -EIO;
1561         }
1562
1563         /* Register debugfs entries for amdgpu_ttm */
1564         amdgpu_ttm_debugfs_init(adev);
1565         amdgpu_debugfs_pm_init(adev);
1566         amdgpu_debugfs_sa_init(adev);
1567         amdgpu_debugfs_fence_init(adev);
1568         amdgpu_debugfs_gem_init(adev);
1569
1570         r = amdgpu_debugfs_regs_init(adev);
1571         if (r)
1572                 DRM_ERROR("registering register debugfs failed (%d).\n", r);
1573
1574         amdgpu_debugfs_firmware_init(adev);
1575
1576 #if defined(CONFIG_DRM_AMD_DC)
1577         if (amdgpu_device_has_dc_support(adev))
1578                 dtn_debugfs_init(adev);
1579 #endif
1580
1581         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
1582                 struct amdgpu_ring *ring = adev->rings[i];
1583
1584                 if (!ring)
1585                         continue;
1586
1587                 if (amdgpu_debugfs_ring_init(adev, ring)) {
1588                         DRM_ERROR("Failed to register debugfs file for rings !\n");
1589                 }
1590         }
1591
1592         amdgpu_ras_debugfs_create_all(adev);
1593         amdgpu_debugfs_autodump_init(adev);
1594         amdgpu_rap_debugfs_init(adev);
1595         amdgpu_securedisplay_debugfs_init(adev);
1596         amdgpu_fw_attestation_debugfs_init(adev);
1597
1598         debugfs_create_file("amdgpu_evict_vram", 0444, root, adev,
1599                             &amdgpu_evict_vram_fops);
1600         debugfs_create_file("amdgpu_evict_gtt", 0444, root, adev,
1601                             &amdgpu_evict_gtt_fops);
1602         debugfs_create_file("amdgpu_test_ib", 0444, root, adev,
1603                             &amdgpu_debugfs_test_ib_fops);
1604         debugfs_create_file("amdgpu_vm_info", 0444, root, adev,
1605                             &amdgpu_debugfs_vm_info_fops);
1606
1607         adev->debugfs_vbios_blob.data = adev->bios;
1608         adev->debugfs_vbios_blob.size = adev->bios_size;
1609         debugfs_create_blob("amdgpu_vbios", 0444, root,
1610                             &adev->debugfs_vbios_blob);
1611
1612         return 0;
1613 }
1614
1615 #else
1616 int amdgpu_debugfs_init(struct amdgpu_device *adev)
1617 {
1618         return 0;
1619 }
1620 int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
1621 {
1622         return 0;
1623 }
1624 #endif