drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c

   1 /*
   2  * Copyright 2008 Advanced Micro Devices, Inc.
   3  * Copyright 2008 Red Hat Inc.
   4  * Copyright 2009 Jerome Glisse.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22  * OTHER DEALINGS IN THE SOFTWARE.
  23  *
  24  */
  25
  26 #include <linux/kthread.h>
  27 #include <linux/pci.h>
  28 #include <linux/uaccess.h>
  29 #include <linux/pm_runtime.h>
  30
  31 #include "amdgpu.h"
  32 #include "amdgpu_pm.h"
  33 #include "amdgpu_dm_debugfs.h"
  34 #include "amdgpu_ras.h"
  35 #include "amdgpu_rap.h"
  36 #include "amdgpu_securedisplay.h"
  37 #include "amdgpu_fw_attestation.h"
  38 #include "amdgpu_umr.h"
  39
  40 #include "amdgpu_reset.h"
  41
  42 #if defined(CONFIG_DEBUG_FS)
  43
  44 /**
  45  * amdgpu_debugfs_process_reg_op - Handle MMIO register reads/writes
  46  *
  47  * @read: True if reading
  48  * @f: open file handle
  49  * @buf: User buffer to write/read to
  50  * @size: Number of bytes to write/read
  51  * @pos:  Offset to seek to
  52  *
  53  * This debugfs entry has special meaning on the offset being sought.
  54  * Various bits have different meanings:
  55  *
  56  * Bit 62:  Indicates a GRBM bank switch is needed
  57  * Bit 61:  Indicates a SRBM bank switch is needed (implies bit 62 is
  58  *          zero)
  59  * Bits 24..33: The SE or ME selector if needed
  60  * Bits 34..43: The SH (or SA) or PIPE selector if needed
  61  * Bits 44..53: The INSTANCE (or CU/WGP) or QUEUE selector if needed
  62  *
  63  * Bit 23:  Indicates that the PM power gating lock should be held
  64  *          This is necessary to read registers that might be
  65  *          unreliable during a power gating transistion.
  66  *
  67  * The lower bits are the BYTE offset of the register to read.  This
  68  * allows reading multiple registers in a single call and having
  69  * the returned size reflect that.
  70  */
  71 static int  amdgpu_debugfs_process_reg_op(bool read, struct file *f,
  72                 char __user *buf, size_t size, loff_t *pos)
  73 {
  74         struct amdgpu_device *adev = file_inode(f)->i_private;
  75         ssize_t result = 0;
  76         int r;
  77         bool pm_pg_lock, use_bank, use_ring;
  78         unsigned instance_bank, sh_bank, se_bank, me, pipe, queue, vmid;
  79
  80         pm_pg_lock = use_bank = use_ring = false;
  81         instance_bank = sh_bank = se_bank = me = pipe = queue = vmid = 0;
  82
  83         if (size & 0x3 || *pos & 0x3 ||
  84                         ((*pos & (1ULL << 62)) && (*pos & (1ULL << 61))))
  85                 return -EINVAL;
  86
  87         /* are we reading registers for which a PG lock is necessary? */
  88         pm_pg_lock = (*pos >> 23) & 1;
  89
  90         if (*pos & (1ULL << 62)) {
  91                 se_bank = (*pos & GENMASK_ULL(33, 24)) >> 24;
  92                 sh_bank = (*pos & GENMASK_ULL(43, 34)) >> 34;
  93                 instance_bank = (*pos & GENMASK_ULL(53, 44)) >> 44;
  94
  95                 if (se_bank == 0x3FF)
  96                         se_bank = 0xFFFFFFFF;
  97                 if (sh_bank == 0x3FF)
  98                         sh_bank = 0xFFFFFFFF;
  99                 if (instance_bank == 0x3FF)
 100                         instance_bank = 0xFFFFFFFF;
 101                 use_bank = true;
 102         } else if (*pos & (1ULL << 61)) {
 103
 104                 me = (*pos & GENMASK_ULL(33, 24)) >> 24;
 105                 pipe = (*pos & GENMASK_ULL(43, 34)) >> 34;
 106                 queue = (*pos & GENMASK_ULL(53, 44)) >> 44;
 107                 vmid = (*pos & GENMASK_ULL(58, 54)) >> 54;
 108
 109                 use_ring = true;
 110         } else {
 111                 use_bank = use_ring = false;
 112         }
 113
 114         *pos &= (1UL << 22) - 1;
 115
 116         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 117         if (r < 0) {
 118                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 119                 return r;
 120         }
 121
 122         r = amdgpu_virt_enable_access_debugfs(adev);
 123         if (r < 0) {
 124                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 125                 return r;
 126         }
 127
 128         if (use_bank) {
 129                 if ((sh_bank != 0xFFFFFFFF && sh_bank >= adev->gfx.config.max_sh_per_se) ||
 130                     (se_bank != 0xFFFFFFFF && se_bank >= adev->gfx.config.max_shader_engines)) {
 131                         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 132                         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 133                         amdgpu_virt_disable_access_debugfs(adev);
 134                         return -EINVAL;
 135                 }
 136                 mutex_lock(&adev->grbm_idx_mutex);
 137                 amdgpu_gfx_select_se_sh(adev, se_bank,
 138                                         sh_bank, instance_bank);
 139         } else if (use_ring) {
 140                 mutex_lock(&adev->srbm_mutex);
 141                 amdgpu_gfx_select_me_pipe_q(adev, me, pipe, queue, vmid);
 142         }
 143
 144         if (pm_pg_lock)
 145                 mutex_lock(&adev->pm.mutex);
 146
 147         while (size) {
 148                 uint32_t value;
 149
 150                 if (read) {
 151                         value = RREG32(*pos >> 2);
 152                         r = put_user(value, (uint32_t *)buf);
 153                 } else {
 154                         r = get_user(value, (uint32_t *)buf);
 155                         if (!r)
 156                                 amdgpu_mm_wreg_mmio_rlc(adev, *pos >> 2, value);
 157                 }
 158                 if (r) {
 159                         result = r;
 160                         goto end;
 161                 }
 162
 163                 result += 4;
 164                 buf += 4;
 165                 *pos += 4;
 166                 size -= 4;
 167         }
 168
 169 end:
 170         if (use_bank) {
 171                 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
 172                 mutex_unlock(&adev->grbm_idx_mutex);
 173         } else if (use_ring) {
 174                 amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0);
 175                 mutex_unlock(&adev->srbm_mutex);
 176         }
 177
 178         if (pm_pg_lock)
 179                 mutex_unlock(&adev->pm.mutex);
 180
 181         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 182         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 183
 184         amdgpu_virt_disable_access_debugfs(adev);
 185         return result;
 186 }
 187
 188 /*
 189  * amdgpu_debugfs_regs_read - Callback for reading MMIO registers
 190  */
 191 static ssize_t amdgpu_debugfs_regs_read(struct file *f, char __user *buf,
 192                                         size_t size, loff_t *pos)
 193 {
 194         return amdgpu_debugfs_process_reg_op(true, f, buf, size, pos);
 195 }
 196
 197 /*
 198  * amdgpu_debugfs_regs_write - Callback for writing MMIO registers
 199  */
 200 static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,
 201                                          size_t size, loff_t *pos)
 202 {
 203         return amdgpu_debugfs_process_reg_op(false, f, (char __user *)buf, size, pos);
 204 }
 205
 206 static int amdgpu_debugfs_regs2_open(struct inode *inode, struct file *file)
 207 {
 208         struct amdgpu_debugfs_regs2_data *rd;
 209
 210         rd = kzalloc(sizeof *rd, GFP_KERNEL);
 211         if (!rd)
 212                 return -ENOMEM;
 213         rd->adev = file_inode(file)->i_private;
 214         file->private_data = rd;
 215         mutex_init(&rd->lock);
 216
 217         return 0;
 218 }
 219
 220 static int amdgpu_debugfs_regs2_release(struct inode *inode, struct file *file)
 221 {
 222         struct amdgpu_debugfs_regs2_data *rd = file->private_data;
 223         mutex_destroy(&rd->lock);
 224         kfree(file->private_data);
 225         return 0;
 226 }
 227
 228 static ssize_t amdgpu_debugfs_regs2_op(struct file *f, char __user *buf, u32 offset, size_t size, int write_en)
 229 {
 230         struct amdgpu_debugfs_regs2_data *rd = f->private_data;
 231         struct amdgpu_device *adev = rd->adev;
 232         ssize_t result = 0;
 233         int r;
 234         uint32_t value;
 235
 236         if (size & 0x3 || offset & 0x3)
 237                 return -EINVAL;
 238
 239         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 240         if (r < 0) {
 241                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 242                 return r;
 243         }
 244
 245         r = amdgpu_virt_enable_access_debugfs(adev);
 246         if (r < 0) {
 247                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 248                 return r;
 249         }
 250
 251         mutex_lock(&rd->lock);
 252
 253         if (rd->id.use_grbm) {
 254                 if ((rd->id.grbm.sh != 0xFFFFFFFF && rd->id.grbm.sh >= adev->gfx.config.max_sh_per_se) ||
 255                     (rd->id.grbm.se != 0xFFFFFFFF && rd->id.grbm.se >= adev->gfx.config.max_shader_engines)) {
 256                         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 257                         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 258                         amdgpu_virt_disable_access_debugfs(adev);
 259                         mutex_unlock(&rd->lock);
 260                         return -EINVAL;
 261                 }
 262                 mutex_lock(&adev->grbm_idx_mutex);
 263                 amdgpu_gfx_select_se_sh(adev, rd->id.grbm.se,
 264                                                                 rd->id.grbm.sh,
 265                                                                 rd->id.grbm.instance);
 266         }
 267
 268         if (rd->id.use_srbm) {
 269                 mutex_lock(&adev->srbm_mutex);
 270                 amdgpu_gfx_select_me_pipe_q(adev, rd->id.srbm.me, rd->id.srbm.pipe,
 271                                                                         rd->id.srbm.queue, rd->id.srbm.vmid);
 272         }
 273
 274         if (rd->id.pg_lock)
 275                 mutex_lock(&adev->pm.mutex);
 276
 277         while (size) {
 278                 if (!write_en) {
 279                         value = RREG32(offset >> 2);
 280                         r = put_user(value, (uint32_t *)buf);
 281                 } else {
 282                         r = get_user(value, (uint32_t *)buf);
 283                         if (!r)
 284                                 amdgpu_mm_wreg_mmio_rlc(adev, offset >> 2, value);
 285                 }
 286                 if (r) {
 287                         result = r;
 288                         goto end;
 289                 }
 290                 offset += 4;
 291                 size -= 4;
 292                 result += 4;
 293                 buf += 4;
 294         }
 295 end:
 296         if (rd->id.use_grbm) {
 297                 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
 298                 mutex_unlock(&adev->grbm_idx_mutex);
 299         }
 300
 301         if (rd->id.use_srbm) {
 302                 amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0);
 303                 mutex_unlock(&adev->srbm_mutex);
 304         }
 305
 306         if (rd->id.pg_lock)
 307                 mutex_unlock(&adev->pm.mutex);
 308
 309         mutex_unlock(&rd->lock);
 310
 311         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 312         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 313
 314         amdgpu_virt_disable_access_debugfs(adev);
 315         return result;
 316 }
 317
 318 static long amdgpu_debugfs_regs2_ioctl(struct file *f, unsigned int cmd, unsigned long data)
 319 {
 320         struct amdgpu_debugfs_regs2_data *rd = f->private_data;
 321         int r;
 322
 323         switch (cmd) {
 324         case AMDGPU_DEBUGFS_REGS2_IOC_SET_STATE:
 325                 mutex_lock(&rd->lock);
 326                 r = copy_from_user(&rd->id, (struct amdgpu_debugfs_regs2_iocdata *)data, sizeof rd->id);
 327                 mutex_unlock(&rd->lock);
 328                 return r ? -EINVAL : 0;
 329         default:
 330                 return -EINVAL;
 331         }
 332         return 0;
 333 }
 334
 335 static ssize_t amdgpu_debugfs_regs2_read(struct file *f, char __user *buf, size_t size, loff_t *pos)
 336 {
 337         return amdgpu_debugfs_regs2_op(f, buf, *pos, size, 0);
 338 }
 339
 340 static ssize_t amdgpu_debugfs_regs2_write(struct file *f, const char __user *buf, size_t size, loff_t *pos)
 341 {
 342         return amdgpu_debugfs_regs2_op(f, (char __user *)buf, *pos, size, 1);
 343 }
 344
 345
 346 /**
 347  * amdgpu_debugfs_regs_pcie_read - Read from a PCIE register
 348  *
 349  * @f: open file handle
 350  * @buf: User buffer to store read data in
 351  * @size: Number of bytes to read
 352  * @pos:  Offset to seek to
 353  *
 354  * The lower bits are the BYTE offset of the register to read.  This
 355  * allows reading multiple registers in a single call and having
 356  * the returned size reflect that.
 357  */
 358 static ssize_t amdgpu_debugfs_regs_pcie_read(struct file *f, char __user *buf,
 359                                         size_t size, loff_t *pos)
 360 {
 361         struct amdgpu_device *adev = file_inode(f)->i_private;
 362         ssize_t result = 0;
 363         int r;
 364
 365         if (size & 0x3 || *pos & 0x3)
 366                 return -EINVAL;
 367
 368         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 369         if (r < 0) {
 370                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 371                 return r;
 372         }
 373
 374         r = amdgpu_virt_enable_access_debugfs(adev);
 375         if (r < 0) {
 376                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 377                 return r;
 378         }
 379
 380         while (size) {
 381                 uint32_t value;
 382
 383                 value = RREG32_PCIE(*pos);
 384                 r = put_user(value, (uint32_t *)buf);
 385                 if (r) {
 386                         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 387                         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 388                         amdgpu_virt_disable_access_debugfs(adev);
 389                         return r;
 390                 }
 391
 392                 result += 4;
 393                 buf += 4;
 394                 *pos += 4;
 395                 size -= 4;
 396         }
 397
 398         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 399         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 400
 401         amdgpu_virt_disable_access_debugfs(adev);
 402         return result;
 403 }
 404
 405 /**
 406  * amdgpu_debugfs_regs_pcie_write - Write to a PCIE register
 407  *
 408  * @f: open file handle
 409  * @buf: User buffer to write data from
 410  * @size: Number of bytes to write
 411  * @pos:  Offset to seek to
 412  *
 413  * The lower bits are the BYTE offset of the register to write.  This
 414  * allows writing multiple registers in a single call and having
 415  * the returned size reflect that.
 416  */
 417 static ssize_t amdgpu_debugfs_regs_pcie_write(struct file *f, const char __user *buf,
 418                                          size_t size, loff_t *pos)
 419 {
 420         struct amdgpu_device *adev = file_inode(f)->i_private;
 421         ssize_t result = 0;
 422         int r;
 423
 424         if (size & 0x3 || *pos & 0x3)
 425                 return -EINVAL;
 426
 427         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 428         if (r < 0) {
 429                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 430                 return r;
 431         }
 432
 433         r = amdgpu_virt_enable_access_debugfs(adev);
 434         if (r < 0) {
 435                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 436                 return r;
 437         }
 438
 439         while (size) {
 440                 uint32_t value;
 441
 442                 r = get_user(value, (uint32_t *)buf);
 443                 if (r) {
 444                         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 445                         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 446                         amdgpu_virt_disable_access_debugfs(adev);
 447                         return r;
 448                 }
 449
 450                 WREG32_PCIE(*pos, value);
 451
 452                 result += 4;
 453                 buf += 4;
 454                 *pos += 4;
 455                 size -= 4;
 456         }
 457
 458         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 459         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 460
 461         amdgpu_virt_disable_access_debugfs(adev);
 462         return result;
 463 }
 464
 465 /**
 466  * amdgpu_debugfs_regs_didt_read - Read from a DIDT register
 467  *
 468  * @f: open file handle
 469  * @buf: User buffer to store read data in
 470  * @size: Number of bytes to read
 471  * @pos:  Offset to seek to
 472  *
 473  * The lower bits are the BYTE offset of the register to read.  This
 474  * allows reading multiple registers in a single call and having
 475  * the returned size reflect that.
 476  */
 477 static ssize_t amdgpu_debugfs_regs_didt_read(struct file *f, char __user *buf,
 478                                         size_t size, loff_t *pos)
 479 {
 480         struct amdgpu_device *adev = file_inode(f)->i_private;
 481         ssize_t result = 0;
 482         int r;
 483
 484         if (size & 0x3 || *pos & 0x3)
 485                 return -EINVAL;
 486
 487         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 488         if (r < 0) {
 489                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 490                 return r;
 491         }
 492
 493         r = amdgpu_virt_enable_access_debugfs(adev);
 494         if (r < 0) {
 495                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 496                 return r;
 497         }
 498
 499         while (size) {
 500                 uint32_t value;
 501
 502                 value = RREG32_DIDT(*pos >> 2);
 503                 r = put_user(value, (uint32_t *)buf);
 504                 if (r) {
 505                         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 506                         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 507                         amdgpu_virt_disable_access_debugfs(adev);
 508                         return r;
 509                 }
 510
 511                 result += 4;
 512                 buf += 4;
 513                 *pos += 4;
 514                 size -= 4;
 515         }
 516
 517         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 518         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 519
 520         amdgpu_virt_disable_access_debugfs(adev);
 521         return result;
 522 }
 523
 524 /**
 525  * amdgpu_debugfs_regs_didt_write - Write to a DIDT register
 526  *
 527  * @f: open file handle
 528  * @buf: User buffer to write data from
 529  * @size: Number of bytes to write
 530  * @pos:  Offset to seek to
 531  *
 532  * The lower bits are the BYTE offset of the register to write.  This
 533  * allows writing multiple registers in a single call and having
 534  * the returned size reflect that.
 535  */
 536 static ssize_t amdgpu_debugfs_regs_didt_write(struct file *f, const char __user *buf,
 537                                          size_t size, loff_t *pos)
 538 {
 539         struct amdgpu_device *adev = file_inode(f)->i_private;
 540         ssize_t result = 0;
 541         int r;
 542
 543         if (size & 0x3 || *pos & 0x3)
 544                 return -EINVAL;
 545
 546         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 547         if (r < 0) {
 548                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 549                 return r;
 550         }
 551
 552         r = amdgpu_virt_enable_access_debugfs(adev);
 553         if (r < 0) {
 554                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 555                 return r;
 556         }
 557
 558         while (size) {
 559                 uint32_t value;
 560
 561                 r = get_user(value, (uint32_t *)buf);
 562                 if (r) {
 563                         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 564                         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 565                         amdgpu_virt_disable_access_debugfs(adev);
 566                         return r;
 567                 }
 568
 569                 WREG32_DIDT(*pos >> 2, value);
 570
 571                 result += 4;
 572                 buf += 4;
 573                 *pos += 4;
 574                 size -= 4;
 575         }
 576
 577         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 578         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 579
 580         amdgpu_virt_disable_access_debugfs(adev);
 581         return result;
 582 }
 583
 584 /**
 585  * amdgpu_debugfs_regs_smc_read - Read from a SMC register
 586  *
 587  * @f: open file handle
 588  * @buf: User buffer to store read data in
 589  * @size: Number of bytes to read
 590  * @pos:  Offset to seek to
 591  *
 592  * The lower bits are the BYTE offset of the register to read.  This
 593  * allows reading multiple registers in a single call and having
 594  * the returned size reflect that.
 595  */
 596 static ssize_t amdgpu_debugfs_regs_smc_read(struct file *f, char __user *buf,
 597                                         size_t size, loff_t *pos)
 598 {
 599         struct amdgpu_device *adev = file_inode(f)->i_private;
 600         ssize_t result = 0;
 601         int r;
 602
 603         if (size & 0x3 || *pos & 0x3)
 604                 return -EINVAL;
 605
 606         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 607         if (r < 0) {
 608                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 609                 return r;
 610         }
 611
 612         r = amdgpu_virt_enable_access_debugfs(adev);
 613         if (r < 0) {
 614                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 615                 return r;
 616         }
 617
 618         while (size) {
 619                 uint32_t value;
 620
 621                 value = RREG32_SMC(*pos);
 622                 r = put_user(value, (uint32_t *)buf);
 623                 if (r) {
 624                         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 625                         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 626                         amdgpu_virt_disable_access_debugfs(adev);
 627                         return r;
 628                 }
 629
 630                 result += 4;
 631                 buf += 4;
 632                 *pos += 4;
 633                 size -= 4;
 634         }
 635
 636         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 637         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 638
 639         amdgpu_virt_disable_access_debugfs(adev);
 640         return result;
 641 }
 642
 643 /**
 644  * amdgpu_debugfs_regs_smc_write - Write to a SMC register
 645  *
 646  * @f: open file handle
 647  * @buf: User buffer to write data from
 648  * @size: Number of bytes to write
 649  * @pos:  Offset to seek to
 650  *
 651  * The lower bits are the BYTE offset of the register to write.  This
 652  * allows writing multiple registers in a single call and having
 653  * the returned size reflect that.
 654  */
 655 static ssize_t amdgpu_debugfs_regs_smc_write(struct file *f, const char __user *buf,
 656                                          size_t size, loff_t *pos)
 657 {
 658         struct amdgpu_device *adev = file_inode(f)->i_private;
 659         ssize_t result = 0;
 660         int r;
 661
 662         if (size & 0x3 || *pos & 0x3)
 663                 return -EINVAL;
 664
 665         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 666         if (r < 0) {
 667                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 668                 return r;
 669         }
 670
 671         r = amdgpu_virt_enable_access_debugfs(adev);
 672         if (r < 0) {
 673                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 674                 return r;
 675         }
 676
 677         while (size) {
 678                 uint32_t value;
 679
 680                 r = get_user(value, (uint32_t *)buf);
 681                 if (r) {
 682                         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 683                         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 684                         amdgpu_virt_disable_access_debugfs(adev);
 685                         return r;
 686                 }
 687
 688                 WREG32_SMC(*pos, value);
 689
 690                 result += 4;
 691                 buf += 4;
 692                 *pos += 4;
 693                 size -= 4;
 694         }
 695
 696         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 697         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 698
 699         amdgpu_virt_disable_access_debugfs(adev);
 700         return result;
 701 }
 702
 703 /**
 704  * amdgpu_debugfs_gca_config_read - Read from gfx config data
 705  *
 706  * @f: open file handle
 707  * @buf: User buffer to store read data in
 708  * @size: Number of bytes to read
 709  * @pos:  Offset to seek to
 710  *
 711  * This file is used to access configuration data in a somewhat
 712  * stable fashion.  The format is a series of DWORDs with the first
 713  * indicating which revision it is.  New content is appended to the
 714  * end so that older software can still read the data.
 715  */
 716
 717 static ssize_t amdgpu_debugfs_gca_config_read(struct file *f, char __user *buf,
 718                                         size_t size, loff_t *pos)
 719 {
 720         struct amdgpu_device *adev = file_inode(f)->i_private;
 721         ssize_t result = 0;
 722         int r;
 723         uint32_t *config, no_regs = 0;
 724
 725         if (size & 0x3 || *pos & 0x3)
 726                 return -EINVAL;
 727
 728         config = kmalloc_array(256, sizeof(*config), GFP_KERNEL);
 729         if (!config)
 730                 return -ENOMEM;
 731
 732         /* version, increment each time something is added */
 733         config[no_regs++] = 4;
 734         config[no_regs++] = adev->gfx.config.max_shader_engines;
 735         config[no_regs++] = adev->gfx.config.max_tile_pipes;
 736         config[no_regs++] = adev->gfx.config.max_cu_per_sh;
 737         config[no_regs++] = adev->gfx.config.max_sh_per_se;
 738         config[no_regs++] = adev->gfx.config.max_backends_per_se;
 739         config[no_regs++] = adev->gfx.config.max_texture_channel_caches;
 740         config[no_regs++] = adev->gfx.config.max_gprs;
 741         config[no_regs++] = adev->gfx.config.max_gs_threads;
 742         config[no_regs++] = adev->gfx.config.max_hw_contexts;
 743         config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_frontend;
 744         config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_backend;
 745         config[no_regs++] = adev->gfx.config.sc_hiz_tile_fifo_size;
 746         config[no_regs++] = adev->gfx.config.sc_earlyz_tile_fifo_size;
 747         config[no_regs++] = adev->gfx.config.num_tile_pipes;
 748         config[no_regs++] = adev->gfx.config.backend_enable_mask;
 749         config[no_regs++] = adev->gfx.config.mem_max_burst_length_bytes;
 750         config[no_regs++] = adev->gfx.config.mem_row_size_in_kb;
 751         config[no_regs++] = adev->gfx.config.shader_engine_tile_size;
 752         config[no_regs++] = adev->gfx.config.num_gpus;
 753         config[no_regs++] = adev->gfx.config.multi_gpu_tile_size;
 754         config[no_regs++] = adev->gfx.config.mc_arb_ramcfg;
 755         config[no_regs++] = adev->gfx.config.gb_addr_config;
 756         config[no_regs++] = adev->gfx.config.num_rbs;
 757
 758         /* rev==1 */
 759         config[no_regs++] = adev->rev_id;
 760         config[no_regs++] = adev->pg_flags;
 761         config[no_regs++] = adev->cg_flags;
 762
 763         /* rev==2 */
 764         config[no_regs++] = adev->family;
 765         config[no_regs++] = adev->external_rev_id;
 766
 767         /* rev==3 */
 768         config[no_regs++] = adev->pdev->device;
 769         config[no_regs++] = adev->pdev->revision;
 770         config[no_regs++] = adev->pdev->subsystem_device;
 771         config[no_regs++] = adev->pdev->subsystem_vendor;
 772
 773         /* rev==4 APU flag */
 774         config[no_regs++] = adev->flags & AMD_IS_APU ? 1 : 0;
 775
 776         while (size && (*pos < no_regs * 4)) {
 777                 uint32_t value;
 778
 779                 value = config[*pos >> 2];
 780                 r = put_user(value, (uint32_t *)buf);
 781                 if (r) {
 782                         kfree(config);
 783                         return r;
 784                 }
 785
 786                 result += 4;
 787                 buf += 4;
 788                 *pos += 4;
 789                 size -= 4;
 790         }
 791
 792         kfree(config);
 793         return result;
 794 }
 795
 796 /**
 797  * amdgpu_debugfs_sensor_read - Read from the powerplay sensors
 798  *
 799  * @f: open file handle
 800  * @buf: User buffer to store read data in
 801  * @size: Number of bytes to read
 802  * @pos:  Offset to seek to
 803  *
 804  * The offset is treated as the BYTE address of one of the sensors
 805  * enumerated in amd/include/kgd_pp_interface.h under the
 806  * 'amd_pp_sensors' enumeration.  For instance to read the UVD VCLK
 807  * you would use the offset 3 * 4 = 12.
 808  */
 809 static ssize_t amdgpu_debugfs_sensor_read(struct file *f, char __user *buf,
 810                                         size_t size, loff_t *pos)
 811 {
 812         struct amdgpu_device *adev = file_inode(f)->i_private;
 813         int idx, x, outsize, r, valuesize;
 814         uint32_t values[16];
 815
 816         if (size & 3 || *pos & 0x3)
 817                 return -EINVAL;
 818
 819         if (!adev->pm.dpm_enabled)
 820                 return -EINVAL;
 821
 822         /* convert offset to sensor number */
 823         idx = *pos >> 2;
 824
 825         valuesize = sizeof(values);
 826
 827         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 828         if (r < 0) {
 829                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 830                 return r;
 831         }
 832
 833         r = amdgpu_virt_enable_access_debugfs(adev);
 834         if (r < 0) {
 835                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 836                 return r;
 837         }
 838
 839         r = amdgpu_dpm_read_sensor(adev, idx, &values[0], &valuesize);
 840
 841         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 842         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 843
 844         if (r) {
 845                 amdgpu_virt_disable_access_debugfs(adev);
 846                 return r;
 847         }
 848
 849         if (size > valuesize) {
 850                 amdgpu_virt_disable_access_debugfs(adev);
 851                 return -EINVAL;
 852         }
 853
 854         outsize = 0;
 855         x = 0;
 856         if (!r) {
 857                 while (size) {
 858                         r = put_user(values[x++], (int32_t *)buf);
 859                         buf += 4;
 860                         size -= 4;
 861                         outsize += 4;
 862                 }
 863         }
 864
 865         amdgpu_virt_disable_access_debugfs(adev);
 866         return !r ? outsize : r;
 867 }
 868
 869 /** amdgpu_debugfs_wave_read - Read WAVE STATUS data
 870  *
 871  * @f: open file handle
 872  * @buf: User buffer to store read data in
 873  * @size: Number of bytes to read
 874  * @pos:  Offset to seek to
 875  *
 876  * The offset being sought changes which wave that the status data
 877  * will be returned for.  The bits are used as follows:
 878  *
 879  * Bits 0..6:   Byte offset into data
 880  * Bits 7..14:  SE selector
 881  * Bits 15..22: SH/SA selector
 882  * Bits 23..30: CU/{WGP+SIMD} selector
 883  * Bits 31..36: WAVE ID selector
 884  * Bits 37..44: SIMD ID selector
 885  *
 886  * The returned data begins with one DWORD of version information
 887  * Followed by WAVE STATUS registers relevant to the GFX IP version
 888  * being used.  See gfx_v8_0_read_wave_data() for an example output.
 889  */
 890 static ssize_t amdgpu_debugfs_wave_read(struct file *f, char __user *buf,
 891                                         size_t size, loff_t *pos)
 892 {
 893         struct amdgpu_device *adev = f->f_inode->i_private;
 894         int r, x;
 895         ssize_t result = 0;
 896         uint32_t offset, se, sh, cu, wave, simd, data[32];
 897
 898         if (size & 3 || *pos & 3)
 899                 return -EINVAL;
 900
 901         /* decode offset */
 902         offset = (*pos & GENMASK_ULL(6, 0));
 903         se = (*pos & GENMASK_ULL(14, 7)) >> 7;
 904         sh = (*pos & GENMASK_ULL(22, 15)) >> 15;
 905         cu = (*pos & GENMASK_ULL(30, 23)) >> 23;
 906         wave = (*pos & GENMASK_ULL(36, 31)) >> 31;
 907         simd = (*pos & GENMASK_ULL(44, 37)) >> 37;
 908
 909         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 910         if (r < 0) {
 911                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 912                 return r;
 913         }
 914
 915         r = amdgpu_virt_enable_access_debugfs(adev);
 916         if (r < 0) {
 917                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 918                 return r;
 919         }
 920
 921         /* switch to the specific se/sh/cu */
 922         mutex_lock(&adev->grbm_idx_mutex);
 923         amdgpu_gfx_select_se_sh(adev, se, sh, cu);
 924
 925         x = 0;
 926         if (adev->gfx.funcs->read_wave_data)
 927                 adev->gfx.funcs->read_wave_data(adev, simd, wave, data, &x);
 928
 929         amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF);
 930         mutex_unlock(&adev->grbm_idx_mutex);
 931
 932         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 933         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 934
 935         if (!x) {
 936                 amdgpu_virt_disable_access_debugfs(adev);
 937                 return -EINVAL;
 938         }
 939
 940         while (size && (offset < x * 4)) {
 941                 uint32_t value;
 942
 943                 value = data[offset >> 2];
 944                 r = put_user(value, (uint32_t *)buf);
 945                 if (r) {
 946                         amdgpu_virt_disable_access_debugfs(adev);
 947                         return r;
 948                 }
 949
 950                 result += 4;
 951                 buf += 4;
 952                 offset += 4;
 953                 size -= 4;
 954         }
 955
 956         amdgpu_virt_disable_access_debugfs(adev);
 957         return result;
 958 }
 959
 960 /** amdgpu_debugfs_gpr_read - Read wave gprs
 961  *
 962  * @f: open file handle
 963  * @buf: User buffer to store read data in
 964  * @size: Number of bytes to read
 965  * @pos:  Offset to seek to
 966  *
 967  * The offset being sought changes which wave that the status data
 968  * will be returned for.  The bits are used as follows:
 969  *
 970  * Bits 0..11:  Byte offset into data
 971  * Bits 12..19: SE selector
 972  * Bits 20..27: SH/SA selector
 973  * Bits 28..35: CU/{WGP+SIMD} selector
 974  * Bits 36..43: WAVE ID selector
 975  * Bits 37..44: SIMD ID selector
 976  * Bits 52..59: Thread selector
 977  * Bits 60..61: Bank selector (VGPR=0,SGPR=1)
 978  *
 979  * The return data comes from the SGPR or VGPR register bank for
 980  * the selected operational unit.
 981  */
 982 static ssize_t amdgpu_debugfs_gpr_read(struct file *f, char __user *buf,
 983                                         size_t size, loff_t *pos)
 984 {
 985         struct amdgpu_device *adev = f->f_inode->i_private;
 986         int r;
 987         ssize_t result = 0;
 988         uint32_t offset, se, sh, cu, wave, simd, thread, bank, *data;
 989
 990         if (size > 4096 || size & 3 || *pos & 3)
 991                 return -EINVAL;
 992
 993         /* decode offset */
 994         offset = (*pos & GENMASK_ULL(11, 0)) >> 2;
 995         se = (*pos & GENMASK_ULL(19, 12)) >> 12;
 996         sh = (*pos & GENMASK_ULL(27, 20)) >> 20;
 997         cu = (*pos & GENMASK_ULL(35, 28)) >> 28;
 998         wave = (*pos & GENMASK_ULL(43, 36)) >> 36;
 999         simd = (*pos & GENMASK_ULL(51, 44)) >> 44;
1000         thread = (*pos & GENMASK_ULL(59, 52)) >> 52;
1001         bank = (*pos & GENMASK_ULL(61, 60)) >> 60;
1002
1003         data = kcalloc(1024, sizeof(*data), GFP_KERNEL);
1004         if (!data)
1005                 return -ENOMEM;
1006
1007         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1008         if (r < 0)
1009                 goto err;
1010
1011         r = amdgpu_virt_enable_access_debugfs(adev);
1012         if (r < 0)
1013                 goto err;
1014
1015         /* switch to the specific se/sh/cu */
1016         mutex_lock(&adev->grbm_idx_mutex);
1017         amdgpu_gfx_select_se_sh(adev, se, sh, cu);
1018
1019         if (bank == 0) {
1020                 if (adev->gfx.funcs->read_wave_vgprs)
1021                         adev->gfx.funcs->read_wave_vgprs(adev, simd, wave, thread, offset, size>>2, data);
1022         } else {
1023                 if (adev->gfx.funcs->read_wave_sgprs)
1024                         adev->gfx.funcs->read_wave_sgprs(adev, simd, wave, offset, size>>2, data);
1025         }
1026
1027         amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF);
1028         mutex_unlock(&adev->grbm_idx_mutex);
1029
1030         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1031         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1032
1033         while (size) {
1034                 uint32_t value;
1035
1036                 value = data[result >> 2];
1037                 r = put_user(value, (uint32_t *)buf);
1038                 if (r) {
1039                         amdgpu_virt_disable_access_debugfs(adev);
1040                         goto err;
1041                 }
1042
1043                 result += 4;
1044                 buf += 4;
1045                 size -= 4;
1046         }
1047
1048         kfree(data);
1049         amdgpu_virt_disable_access_debugfs(adev);
1050         return result;
1051
1052 err:
1053         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1054         kfree(data);
1055         return r;
1056 }
1057
1058 /**
1059  * amdgpu_debugfs_gfxoff_write - Enable/disable GFXOFF
1060  *
1061  * @f: open file handle
1062  * @buf: User buffer to write data from
1063  * @size: Number of bytes to write
1064  * @pos:  Offset to seek to
1065  *
1066  * Write a 32-bit zero to disable or a 32-bit non-zero to enable
1067  */
1068 static ssize_t amdgpu_debugfs_gfxoff_write(struct file *f, const char __user *buf,
1069                                          size_t size, loff_t *pos)
1070 {
1071         struct amdgpu_device *adev = file_inode(f)->i_private;
1072         ssize_t result = 0;
1073         int r;
1074
1075         if (size & 0x3 || *pos & 0x3)
1076                 return -EINVAL;
1077
1078         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1079         if (r < 0) {
1080                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1081                 return r;
1082         }
1083
1084         while (size) {
1085                 uint32_t value;
1086
1087                 r = get_user(value, (uint32_t *)buf);
1088                 if (r) {
1089                         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1090                         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1091                         return r;
1092                 }
1093
1094                 amdgpu_gfx_off_ctrl(adev, value ? true : false);
1095
1096                 result += 4;
1097                 buf += 4;
1098                 *pos += 4;
1099                 size -= 4;
1100         }
1101
1102         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1103         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1104
1105         return result;
1106 }
1107
1108
1109 /**
1110  * amdgpu_debugfs_gfxoff_read - read gfxoff status
1111  *
1112  * @f: open file handle
1113  * @buf: User buffer to store read data in
1114  * @size: Number of bytes to read
1115  * @pos:  Offset to seek to
1116  */
1117 static ssize_t amdgpu_debugfs_gfxoff_read(struct file *f, char __user *buf,
1118                                          size_t size, loff_t *pos)
1119 {
1120         struct amdgpu_device *adev = file_inode(f)->i_private;
1121         ssize_t result = 0;
1122         int r;
1123
1124         if (size & 0x3 || *pos & 0x3)
1125                 return -EINVAL;
1126
1127         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1128         if (r < 0) {
1129                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1130                 return r;
1131         }
1132
1133         while (size) {
1134                 uint32_t value;
1135
1136                 r = amdgpu_get_gfx_off_status(adev, &value);
1137                 if (r) {
1138                         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1139                         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1140                         return r;
1141                 }
1142
1143                 r = put_user(value, (uint32_t *)buf);
1144                 if (r) {
1145                         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1146                         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1147                         return r;
1148                 }
1149
1150                 result += 4;
1151                 buf += 4;
1152                 *pos += 4;
1153                 size -= 4;
1154         }
1155
1156         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1157         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1158
1159         return result;
1160 }
1161
1162 static const struct file_operations amdgpu_debugfs_regs2_fops = {
1163         .owner = THIS_MODULE,
1164         .unlocked_ioctl = amdgpu_debugfs_regs2_ioctl,
1165         .read = amdgpu_debugfs_regs2_read,
1166         .write = amdgpu_debugfs_regs2_write,
1167         .open = amdgpu_debugfs_regs2_open,
1168         .release = amdgpu_debugfs_regs2_release,
1169         .llseek = default_llseek
1170 };
1171
1172 static const struct file_operations amdgpu_debugfs_regs_fops = {
1173         .owner = THIS_MODULE,
1174         .read = amdgpu_debugfs_regs_read,
1175         .write = amdgpu_debugfs_regs_write,
1176         .llseek = default_llseek
1177 };
1178 static const struct file_operations amdgpu_debugfs_regs_didt_fops = {
1179         .owner = THIS_MODULE,
1180         .read = amdgpu_debugfs_regs_didt_read,
1181         .write = amdgpu_debugfs_regs_didt_write,
1182         .llseek = default_llseek
1183 };
1184 static const struct file_operations amdgpu_debugfs_regs_pcie_fops = {
1185         .owner = THIS_MODULE,
1186         .read = amdgpu_debugfs_regs_pcie_read,
1187         .write = amdgpu_debugfs_regs_pcie_write,
1188         .llseek = default_llseek
1189 };
1190 static const struct file_operations amdgpu_debugfs_regs_smc_fops = {
1191         .owner = THIS_MODULE,
1192         .read = amdgpu_debugfs_regs_smc_read,
1193         .write = amdgpu_debugfs_regs_smc_write,
1194         .llseek = default_llseek
1195 };
1196
1197 static const struct file_operations amdgpu_debugfs_gca_config_fops = {
1198         .owner = THIS_MODULE,
1199         .read = amdgpu_debugfs_gca_config_read,
1200         .llseek = default_llseek
1201 };
1202
1203 static const struct file_operations amdgpu_debugfs_sensors_fops = {
1204         .owner = THIS_MODULE,
1205         .read = amdgpu_debugfs_sensor_read,
1206         .llseek = default_llseek
1207 };
1208
1209 static const struct file_operations amdgpu_debugfs_wave_fops = {
1210         .owner = THIS_MODULE,
1211         .read = amdgpu_debugfs_wave_read,
1212         .llseek = default_llseek
1213 };
1214 static const struct file_operations amdgpu_debugfs_gpr_fops = {
1215         .owner = THIS_MODULE,
1216         .read = amdgpu_debugfs_gpr_read,
1217         .llseek = default_llseek
1218 };
1219
1220 static const struct file_operations amdgpu_debugfs_gfxoff_fops = {
1221         .owner = THIS_MODULE,
1222         .read = amdgpu_debugfs_gfxoff_read,
1223         .write = amdgpu_debugfs_gfxoff_write,
1224         .llseek = default_llseek
1225 };
1226
1227 static const struct file_operations *debugfs_regs[] = {
1228         &amdgpu_debugfs_regs_fops,
1229         &amdgpu_debugfs_regs2_fops,
1230         &amdgpu_debugfs_regs_didt_fops,
1231         &amdgpu_debugfs_regs_pcie_fops,
1232         &amdgpu_debugfs_regs_smc_fops,
1233         &amdgpu_debugfs_gca_config_fops,
1234         &amdgpu_debugfs_sensors_fops,
1235         &amdgpu_debugfs_wave_fops,
1236         &amdgpu_debugfs_gpr_fops,
1237         &amdgpu_debugfs_gfxoff_fops,
1238 };
1239
1240 static const char *debugfs_regs_names[] = {
1241         "amdgpu_regs",
1242         "amdgpu_regs2",
1243         "amdgpu_regs_didt",
1244         "amdgpu_regs_pcie",
1245         "amdgpu_regs_smc",
1246         "amdgpu_gca_config",
1247         "amdgpu_sensors",
1248         "amdgpu_wave",
1249         "amdgpu_gpr",
1250         "amdgpu_gfxoff",
1251 };
1252
1253 /**
1254  * amdgpu_debugfs_regs_init -   Initialize debugfs entries that provide
1255  *                              register access.
1256  *
1257  * @adev: The device to attach the debugfs entries to
1258  */
1259 int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
1260 {
1261         struct drm_minor *minor = adev_to_drm(adev)->primary;
1262         struct dentry *ent, *root = minor->debugfs_root;
1263         unsigned int i;
1264
1265         for (i = 0; i < ARRAY_SIZE(debugfs_regs); i++) {
1266                 ent = debugfs_create_file(debugfs_regs_names[i],
1267                                           S_IFREG | S_IRUGO, root,
1268                                           adev, debugfs_regs[i]);
1269                 if (!i && !IS_ERR_OR_NULL(ent))
1270                         i_size_write(ent->d_inode, adev->rmmio_size);
1271         }
1272
1273         return 0;
1274 }
1275
1276 static int amdgpu_debugfs_test_ib_show(struct seq_file *m, void *unused)
1277 {
1278         struct amdgpu_device *adev = (struct amdgpu_device *)m->private;
1279         struct drm_device *dev = adev_to_drm(adev);
1280         int r = 0, i;
1281
1282         r = pm_runtime_get_sync(dev->dev);
1283         if (r < 0) {
1284                 pm_runtime_put_autosuspend(dev->dev);
1285                 return r;
1286         }
1287
1288         /* Avoid accidently unparking the sched thread during GPU reset */
1289         r = down_write_killable(&adev->reset_domain->sem);
1290         if (r)
1291                 return r;
1292
1293         /* hold on the scheduler */
1294         for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
1295                 struct amdgpu_ring *ring = adev->rings[i];
1296
1297                 if (!ring || !ring->sched.thread)
1298                         continue;
1299                 kthread_park(ring->sched.thread);
1300         }
1301
1302         seq_printf(m, "run ib test:\n");
1303         r = amdgpu_ib_ring_tests(adev);
1304         if (r)
1305                 seq_printf(m, "ib ring tests failed (%d).\n", r);
1306         else
1307                 seq_printf(m, "ib ring tests passed.\n");
1308
1309         /* go on the scheduler */
1310         for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
1311                 struct amdgpu_ring *ring = adev->rings[i];
1312
1313                 if (!ring || !ring->sched.thread)
1314                         continue;
1315                 kthread_unpark(ring->sched.thread);
1316         }
1317
1318         up_write(&adev->reset_domain->sem);
1319
1320         pm_runtime_mark_last_busy(dev->dev);
1321         pm_runtime_put_autosuspend(dev->dev);
1322
1323         return 0;
1324 }
1325
1326 static int amdgpu_debugfs_evict_vram(void *data, u64 *val)
1327 {
1328         struct amdgpu_device *adev = (struct amdgpu_device *)data;
1329         struct drm_device *dev = adev_to_drm(adev);
1330         int r;
1331
1332         r = pm_runtime_get_sync(dev->dev);
1333         if (r < 0) {
1334                 pm_runtime_put_autosuspend(dev->dev);
1335                 return r;
1336         }
1337
1338         *val = amdgpu_ttm_evict_resources(adev, TTM_PL_VRAM);
1339
1340         pm_runtime_mark_last_busy(dev->dev);
1341         pm_runtime_put_autosuspend(dev->dev);
1342
1343         return 0;
1344 }
1345
1346
1347 static int amdgpu_debugfs_evict_gtt(void *data, u64 *val)
1348 {
1349         struct amdgpu_device *adev = (struct amdgpu_device *)data;
1350         struct drm_device *dev = adev_to_drm(adev);
1351         int r;
1352
1353         r = pm_runtime_get_sync(dev->dev);
1354         if (r < 0) {
1355                 pm_runtime_put_autosuspend(dev->dev);
1356                 return r;
1357         }
1358
1359         *val = amdgpu_ttm_evict_resources(adev, TTM_PL_TT);
1360
1361         pm_runtime_mark_last_busy(dev->dev);
1362         pm_runtime_put_autosuspend(dev->dev);
1363
1364         return 0;
1365 }
1366
1367 static int amdgpu_debugfs_benchmark(void *data, u64 val)
1368 {
1369         struct amdgpu_device *adev = (struct amdgpu_device *)data;
1370         struct drm_device *dev = adev_to_drm(adev);
1371         int r;
1372
1373         r = pm_runtime_get_sync(dev->dev);
1374         if (r < 0) {
1375                 pm_runtime_put_autosuspend(dev->dev);
1376                 return r;
1377         }
1378
1379         r = amdgpu_benchmark(adev, val);
1380
1381         pm_runtime_mark_last_busy(dev->dev);
1382         pm_runtime_put_autosuspend(dev->dev);
1383
1384         return r;
1385 }
1386
1387 static int amdgpu_debugfs_vm_info_show(struct seq_file *m, void *unused)
1388 {
1389         struct amdgpu_device *adev = (struct amdgpu_device *)m->private;
1390         struct drm_device *dev = adev_to_drm(adev);
1391         struct drm_file *file;
1392         int r;
1393
1394         r = mutex_lock_interruptible(&dev->filelist_mutex);
1395         if (r)
1396                 return r;
1397
1398         list_for_each_entry(file, &dev->filelist, lhead) {
1399                 struct amdgpu_fpriv *fpriv = file->driver_priv;
1400                 struct amdgpu_vm *vm = &fpriv->vm;
1401
1402                 seq_printf(m, "pid:%d\tProcess:%s ----------\n",
1403                                 vm->task_info.pid, vm->task_info.process_name);
1404                 r = amdgpu_bo_reserve(vm->root.bo, true);
1405                 if (r)
1406                         break;
1407                 amdgpu_debugfs_vm_bo_info(vm, m);
1408                 amdgpu_bo_unreserve(vm->root.bo);
1409         }
1410
1411         mutex_unlock(&dev->filelist_mutex);
1412
1413         return r;
1414 }
1415
1416 DEFINE_SHOW_ATTRIBUTE(amdgpu_debugfs_test_ib);
1417 DEFINE_SHOW_ATTRIBUTE(amdgpu_debugfs_vm_info);
1418 DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_evict_vram_fops, amdgpu_debugfs_evict_vram,
1419                          NULL, "%lld\n");
1420 DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_evict_gtt_fops, amdgpu_debugfs_evict_gtt,
1421                          NULL, "%lld\n");
1422 DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_benchmark_fops, NULL, amdgpu_debugfs_benchmark,
1423                          "%lld\n");
1424
1425 static void amdgpu_ib_preempt_fences_swap(struct amdgpu_ring *ring,
1426                                           struct dma_fence **fences)
1427 {
1428         struct amdgpu_fence_driver *drv = &ring->fence_drv;
1429         uint32_t sync_seq, last_seq;
1430
1431         last_seq = atomic_read(&ring->fence_drv.last_seq);
1432         sync_seq = ring->fence_drv.sync_seq;
1433
1434         last_seq &= drv->num_fences_mask;
1435         sync_seq &= drv->num_fences_mask;
1436
1437         do {
1438                 struct dma_fence *fence, **ptr;
1439
1440                 ++last_seq;
1441                 last_seq &= drv->num_fences_mask;
1442                 ptr = &drv->fences[last_seq];
1443
1444                 fence = rcu_dereference_protected(*ptr, 1);
1445                 RCU_INIT_POINTER(*ptr, NULL);
1446
1447                 if (!fence)
1448                         continue;
1449
1450                 fences[last_seq] = fence;
1451
1452         } while (last_seq != sync_seq);
1453 }
1454
1455 static void amdgpu_ib_preempt_signal_fences(struct dma_fence **fences,
1456                                             int length)
1457 {
1458         int i;
1459         struct dma_fence *fence;
1460
1461         for (i = 0; i < length; i++) {
1462                 fence = fences[i];
1463                 if (!fence)
1464                         continue;
1465                 dma_fence_signal(fence);
1466                 dma_fence_put(fence);
1467         }
1468 }
1469
1470 static void amdgpu_ib_preempt_job_recovery(struct drm_gpu_scheduler *sched)
1471 {
1472         struct drm_sched_job *s_job;
1473         struct dma_fence *fence;
1474
1475         spin_lock(&sched->job_list_lock);
1476         list_for_each_entry(s_job, &sched->pending_list, list) {
1477                 fence = sched->ops->run_job(s_job);
1478                 dma_fence_put(fence);
1479         }
1480         spin_unlock(&sched->job_list_lock);
1481 }
1482
1483 static void amdgpu_ib_preempt_mark_partial_job(struct amdgpu_ring *ring)
1484 {
1485         struct amdgpu_job *job;
1486         struct drm_sched_job *s_job, *tmp;
1487         uint32_t preempt_seq;
1488         struct dma_fence *fence, **ptr;
1489         struct amdgpu_fence_driver *drv = &ring->fence_drv;
1490         struct drm_gpu_scheduler *sched = &ring->sched;
1491         bool preempted = true;
1492
1493         if (ring->funcs->type != AMDGPU_RING_TYPE_GFX)
1494                 return;
1495
1496         preempt_seq = le32_to_cpu(*(drv->cpu_addr + 2));
1497         if (preempt_seq <= atomic_read(&drv->last_seq)) {
1498                 preempted = false;
1499                 goto no_preempt;
1500         }
1501
1502         preempt_seq &= drv->num_fences_mask;
1503         ptr = &drv->fences[preempt_seq];
1504         fence = rcu_dereference_protected(*ptr, 1);
1505
1506 no_preempt:
1507         spin_lock(&sched->job_list_lock);
1508         list_for_each_entry_safe(s_job, tmp, &sched->pending_list, list) {
1509                 if (dma_fence_is_signaled(&s_job->s_fence->finished)) {
1510                         /* remove job from ring_mirror_list */
1511                         list_del_init(&s_job->list);
1512                         sched->ops->free_job(s_job);
1513                         continue;
1514                 }
1515                 job = to_amdgpu_job(s_job);
1516                 if (preempted && (&job->hw_fence) == fence)
1517                         /* mark the job as preempted */
1518                         job->preemption_status |= AMDGPU_IB_PREEMPTED;
1519         }
1520         spin_unlock(&sched->job_list_lock);
1521 }
1522
1523 static int amdgpu_debugfs_ib_preempt(void *data, u64 val)
1524 {
1525         int r, resched, length;
1526         struct amdgpu_ring *ring;
1527         struct dma_fence **fences = NULL;
1528         struct amdgpu_device *adev = (struct amdgpu_device *)data;
1529
1530         if (val >= AMDGPU_MAX_RINGS)
1531                 return -EINVAL;
1532
1533         ring = adev->rings[val];
1534
1535         if (!ring || !ring->funcs->preempt_ib || !ring->sched.thread)
1536                 return -EINVAL;
1537
1538         /* the last preemption failed */
1539         if (ring->trail_seq != le32_to_cpu(*ring->trail_fence_cpu_addr))
1540                 return -EBUSY;
1541
1542         length = ring->fence_drv.num_fences_mask + 1;
1543         fences = kcalloc(length, sizeof(void *), GFP_KERNEL);
1544         if (!fences)
1545                 return -ENOMEM;
1546
1547         /* Avoid accidently unparking the sched thread during GPU reset */
1548         r = down_read_killable(&adev->reset_domain->sem);
1549         if (r)
1550                 goto pro_end;
1551
1552         /* stop the scheduler */
1553         kthread_park(ring->sched.thread);
1554
1555         resched = ttm_bo_lock_delayed_workqueue(&adev->mman.bdev);
1556
1557         /* preempt the IB */
1558         r = amdgpu_ring_preempt_ib(ring);
1559         if (r) {
1560                 DRM_WARN("failed to preempt ring %d\n", ring->idx);
1561                 goto failure;
1562         }
1563
1564         amdgpu_fence_process(ring);
1565
1566         if (atomic_read(&ring->fence_drv.last_seq) !=
1567             ring->fence_drv.sync_seq) {
1568                 DRM_INFO("ring %d was preempted\n", ring->idx);
1569
1570                 amdgpu_ib_preempt_mark_partial_job(ring);
1571
1572                 /* swap out the old fences */
1573                 amdgpu_ib_preempt_fences_swap(ring, fences);
1574
1575                 amdgpu_fence_driver_force_completion(ring);
1576
1577                 /* resubmit unfinished jobs */
1578                 amdgpu_ib_preempt_job_recovery(&ring->sched);
1579
1580                 /* wait for jobs finished */
1581                 amdgpu_fence_wait_empty(ring);
1582
1583                 /* signal the old fences */
1584                 amdgpu_ib_preempt_signal_fences(fences, length);
1585         }
1586
1587 failure:
1588         /* restart the scheduler */
1589         kthread_unpark(ring->sched.thread);
1590
1591         up_read(&adev->reset_domain->sem);
1592
1593         ttm_bo_unlock_delayed_workqueue(&adev->mman.bdev, resched);
1594
1595 pro_end:
1596         kfree(fences);
1597
1598         return r;
1599 }
1600
1601 static int amdgpu_debugfs_sclk_set(void *data, u64 val)
1602 {
1603         int ret = 0;
1604         uint32_t max_freq, min_freq;
1605         struct amdgpu_device *adev = (struct amdgpu_device *)data;
1606
1607         if (amdgpu_sriov_vf(adev) && !amdgpu_sriov_is_pp_one_vf(adev))
1608                 return -EINVAL;
1609
1610         ret = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1611         if (ret < 0) {
1612                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1613                 return ret;
1614         }
1615
1616         ret = amdgpu_dpm_get_dpm_freq_range(adev, PP_SCLK, &min_freq, &max_freq);
1617         if (ret == -EOPNOTSUPP) {
1618                 ret = 0;
1619                 goto out;
1620         }
1621         if (ret || val > max_freq || val < min_freq) {
1622                 ret = -EINVAL;
1623                 goto out;
1624         }
1625
1626         ret = amdgpu_dpm_set_soft_freq_range(adev, PP_SCLK, (uint32_t)val, (uint32_t)val);
1627         if (ret)
1628                 ret = -EINVAL;
1629
1630 out:
1631         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1632         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1633
1634         return ret;
1635 }
1636
1637 DEFINE_DEBUGFS_ATTRIBUTE(fops_ib_preempt, NULL,
1638                         amdgpu_debugfs_ib_preempt, "%llu\n");
1639
1640 DEFINE_DEBUGFS_ATTRIBUTE(fops_sclk_set, NULL,
1641                         amdgpu_debugfs_sclk_set, "%llu\n");
1642
1643 static ssize_t amdgpu_reset_dump_register_list_read(struct file *f,
1644                                 char __user *buf, size_t size, loff_t *pos)
1645 {
1646         struct amdgpu_device *adev = (struct amdgpu_device *)file_inode(f)->i_private;
1647         char reg_offset[12];
1648         int i, ret, len = 0;
1649
1650         if (*pos)
1651                 return 0;
1652
1653         memset(reg_offset, 0, 12);
1654         ret = down_read_killable(&adev->reset_domain->sem);
1655         if (ret)
1656                 return ret;
1657
1658         for (i = 0; i < adev->num_regs; i++) {
1659                 sprintf(reg_offset, "0x%x\n", adev->reset_dump_reg_list[i]);
1660                 up_read(&adev->reset_domain->sem);
1661                 if (copy_to_user(buf + len, reg_offset, strlen(reg_offset)))
1662                         return -EFAULT;
1663
1664                 len += strlen(reg_offset);
1665                 ret = down_read_killable(&adev->reset_domain->sem);
1666                 if (ret)
1667                         return ret;
1668         }
1669
1670         up_read(&adev->reset_domain->sem);
1671         *pos += len;
1672
1673         return len;
1674 }
1675
1676 static ssize_t amdgpu_reset_dump_register_list_write(struct file *f,
1677                         const char __user *buf, size_t size, loff_t *pos)
1678 {
1679         struct amdgpu_device *adev = (struct amdgpu_device *)file_inode(f)->i_private;
1680         char reg_offset[11];
1681         uint32_t *new, *tmp = NULL;
1682         int ret, i = 0, len = 0;
1683
1684         do {
1685                 memset(reg_offset, 0, 11);
1686                 if (copy_from_user(reg_offset, buf + len,
1687                                         min(10, ((int)size-len)))) {
1688                         ret = -EFAULT;
1689                         goto error_free;
1690                 }
1691
1692                 new = krealloc_array(tmp, i + 1, sizeof(uint32_t), GFP_KERNEL);
1693                 if (!new) {
1694                         ret = -ENOMEM;
1695                         goto error_free;
1696                 }
1697                 tmp = new;
1698                 if (sscanf(reg_offset, "%X %n", &tmp[i], &ret) != 1) {
1699                         ret = -EINVAL;
1700                         goto error_free;
1701                 }
1702
1703                 len += ret;
1704                 i++;
1705         } while (len < size);
1706
1707         ret = down_write_killable(&adev->reset_domain->sem);
1708         if (ret)
1709                 goto error_free;
1710
1711         swap(adev->reset_dump_reg_list, tmp);
1712         adev->num_regs = i;
1713         up_write(&adev->reset_domain->sem);
1714         ret = size;
1715
1716 error_free:
1717         kfree(tmp);
1718         return ret;
1719 }
1720
1721 static const struct file_operations amdgpu_reset_dump_register_list = {
1722         .owner = THIS_MODULE,
1723         .read = amdgpu_reset_dump_register_list_read,
1724         .write = amdgpu_reset_dump_register_list_write,
1725         .llseek = default_llseek
1726 };
1727
1728 int amdgpu_debugfs_init(struct amdgpu_device *adev)
1729 {
1730         struct dentry *root = adev_to_drm(adev)->primary->debugfs_root;
1731         struct dentry *ent;
1732         int r, i;
1733
1734         if (!debugfs_initialized())
1735                 return 0;
1736
1737         debugfs_create_x32("amdgpu_smu_debug", 0600, root,
1738                            &adev->pm.smu_debug_mask);
1739
1740         ent = debugfs_create_file("amdgpu_preempt_ib", 0600, root, adev,
1741                                   &fops_ib_preempt);
1742         if (IS_ERR(ent)) {
1743                 DRM_ERROR("unable to create amdgpu_preempt_ib debugsfs file\n");
1744                 return PTR_ERR(ent);
1745         }
1746
1747         ent = debugfs_create_file("amdgpu_force_sclk", 0200, root, adev,
1748                                   &fops_sclk_set);
1749         if (IS_ERR(ent)) {
1750                 DRM_ERROR("unable to create amdgpu_set_sclk debugsfs file\n");
1751                 return PTR_ERR(ent);
1752         }
1753
1754         /* Register debugfs entries for amdgpu_ttm */
1755         amdgpu_ttm_debugfs_init(adev);
1756         amdgpu_debugfs_pm_init(adev);
1757         amdgpu_debugfs_sa_init(adev);
1758         amdgpu_debugfs_fence_init(adev);
1759         amdgpu_debugfs_gem_init(adev);
1760
1761         r = amdgpu_debugfs_regs_init(adev);
1762         if (r)
1763                 DRM_ERROR("registering register debugfs failed (%d).\n", r);
1764
1765         amdgpu_debugfs_firmware_init(adev);
1766
1767 #if defined(CONFIG_DRM_AMD_DC)
1768         if (amdgpu_device_has_dc_support(adev))
1769                 dtn_debugfs_init(adev);
1770 #endif
1771
1772         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
1773                 struct amdgpu_ring *ring = adev->rings[i];
1774
1775                 if (!ring)
1776                         continue;
1777
1778                 amdgpu_debugfs_ring_init(adev, ring);
1779         }
1780
1781         for ( i = 0; i < adev->vcn.num_vcn_inst; i++) {
1782                 if (!amdgpu_vcnfw_log)
1783                         break;
1784
1785                 if (adev->vcn.harvest_config & (1 << i))
1786                         continue;
1787
1788                 amdgpu_debugfs_vcn_fwlog_init(adev, i, &adev->vcn.inst[i]);
1789         }
1790
1791         amdgpu_ras_debugfs_create_all(adev);
1792         amdgpu_rap_debugfs_init(adev);
1793         amdgpu_securedisplay_debugfs_init(adev);
1794         amdgpu_fw_attestation_debugfs_init(adev);
1795
1796         debugfs_create_file("amdgpu_evict_vram", 0444, root, adev,
1797                             &amdgpu_evict_vram_fops);
1798         debugfs_create_file("amdgpu_evict_gtt", 0444, root, adev,
1799                             &amdgpu_evict_gtt_fops);
1800         debugfs_create_file("amdgpu_test_ib", 0444, root, adev,
1801                             &amdgpu_debugfs_test_ib_fops);
1802         debugfs_create_file("amdgpu_vm_info", 0444, root, adev,
1803                             &amdgpu_debugfs_vm_info_fops);
1804         debugfs_create_file("amdgpu_benchmark", 0200, root, adev,
1805                             &amdgpu_benchmark_fops);
1806         debugfs_create_file("amdgpu_reset_dump_register_list", 0644, root, adev,
1807                             &amdgpu_reset_dump_register_list);
1808
1809         adev->debugfs_vbios_blob.data = adev->bios;
1810         adev->debugfs_vbios_blob.size = adev->bios_size;
1811         debugfs_create_blob("amdgpu_vbios", 0444, root,
1812                             &adev->debugfs_vbios_blob);
1813
1814         adev->debugfs_discovery_blob.data = adev->mman.discovery_bin;
1815         adev->debugfs_discovery_blob.size = adev->mman.discovery_tmr_size;
1816         debugfs_create_blob("amdgpu_discovery", 0444, root,
1817                             &adev->debugfs_discovery_blob);
1818
1819         return 0;
1820 }
1821
1822 #else
1823 int amdgpu_debugfs_init(struct amdgpu_device *adev)
1824 {
1825         return 0;
1826 }
1827 int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
1828 {
1829         return 0;
1830 }
1831 #endif