drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c

   1 /*
   2  * Copyright 2008 Advanced Micro Devices, Inc.
   3  * Copyright 2008 Red Hat Inc.
   4  * Copyright 2009 Jerome Glisse.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22  * OTHER DEALINGS IN THE SOFTWARE.
  23  *
  24  */
  25
  26 #include <linux/kthread.h>
  27 #include <linux/pci.h>
  28 #include <linux/uaccess.h>
  29 #include <linux/pm_runtime.h>
  30 #include <linux/poll.h>
  31
  32 #include "amdgpu.h"
  33 #include "amdgpu_pm.h"
  34 #include "amdgpu_dm_debugfs.h"
  35 #include "amdgpu_ras.h"
  36 #include "amdgpu_rap.h"
  37 #include "amdgpu_securedisplay.h"
  38 #include "amdgpu_fw_attestation.h"
  39 #include "amdgpu_umr.h"
  40
  41 int amdgpu_debugfs_wait_dump(struct amdgpu_device *adev)
  42 {
  43 #if defined(CONFIG_DEBUG_FS)
  44         unsigned long timeout = 600 * HZ;
  45         int ret;
  46
  47         wake_up_interruptible(&adev->autodump.gpu_hang);
  48
  49         ret = wait_for_completion_interruptible_timeout(&adev->autodump.dumping, timeout);
  50         if (ret == 0) {
  51                 pr_err("autodump: timeout, move on to gpu recovery\n");
  52                 return -ETIMEDOUT;
  53         }
  54 #endif
  55         return 0;
  56 }
  57
  58 #if defined(CONFIG_DEBUG_FS)
  59
  60 static int amdgpu_debugfs_autodump_open(struct inode *inode, struct file *file)
  61 {
  62         struct amdgpu_device *adev = inode->i_private;
  63         int ret;
  64
  65         file->private_data = adev;
  66
  67         ret = down_read_killable(&adev->reset_sem);
  68         if (ret)
  69                 return ret;
  70
  71         if (adev->autodump.dumping.done) {
  72                 reinit_completion(&adev->autodump.dumping);
  73                 ret = 0;
  74         } else {
  75                 ret = -EBUSY;
  76         }
  77
  78         up_read(&adev->reset_sem);
  79
  80         return ret;
  81 }
  82
  83 static int amdgpu_debugfs_autodump_release(struct inode *inode, struct file *file)
  84 {
  85         struct amdgpu_device *adev = file->private_data;
  86
  87         complete_all(&adev->autodump.dumping);
  88         return 0;
  89 }
  90
  91 static unsigned int amdgpu_debugfs_autodump_poll(struct file *file, struct poll_table_struct *poll_table)
  92 {
  93         struct amdgpu_device *adev = file->private_data;
  94
  95         poll_wait(file, &adev->autodump.gpu_hang, poll_table);
  96
  97         if (amdgpu_in_reset(adev))
  98                 return POLLIN | POLLRDNORM | POLLWRNORM;
  99
 100         return 0;
 101 }
 102
 103 static const struct file_operations autodump_debug_fops = {
 104         .owner = THIS_MODULE,
 105         .open = amdgpu_debugfs_autodump_open,
 106         .poll = amdgpu_debugfs_autodump_poll,
 107         .release = amdgpu_debugfs_autodump_release,
 108 };
 109
 110 static void amdgpu_debugfs_autodump_init(struct amdgpu_device *adev)
 111 {
 112         init_completion(&adev->autodump.dumping);
 113         complete_all(&adev->autodump.dumping);
 114         init_waitqueue_head(&adev->autodump.gpu_hang);
 115
 116         debugfs_create_file("amdgpu_autodump", 0600,
 117                 adev_to_drm(adev)->primary->debugfs_root,
 118                 adev, &autodump_debug_fops);
 119 }
 120
 121 /**
 122  * amdgpu_debugfs_process_reg_op - Handle MMIO register reads/writes
 123  *
 124  * @read: True if reading
 125  * @f: open file handle
 126  * @buf: User buffer to write/read to
 127  * @size: Number of bytes to write/read
 128  * @pos:  Offset to seek to
 129  *
 130  * This debugfs entry has special meaning on the offset being sought.
 131  * Various bits have different meanings:
 132  *
 133  * Bit 62:  Indicates a GRBM bank switch is needed
 134  * Bit 61:  Indicates a SRBM bank switch is needed (implies bit 62 is
 135  *          zero)
 136  * Bits 24..33: The SE or ME selector if needed
 137  * Bits 34..43: The SH (or SA) or PIPE selector if needed
 138  * Bits 44..53: The INSTANCE (or CU/WGP) or QUEUE selector if needed
 139  *
 140  * Bit 23:  Indicates that the PM power gating lock should be held
 141  *          This is necessary to read registers that might be
 142  *          unreliable during a power gating transistion.
 143  *
 144  * The lower bits are the BYTE offset of the register to read.  This
 145  * allows reading multiple registers in a single call and having
 146  * the returned size reflect that.
 147  */
 148 static int  amdgpu_debugfs_process_reg_op(bool read, struct file *f,
 149                 char __user *buf, size_t size, loff_t *pos)
 150 {
 151         struct amdgpu_device *adev = file_inode(f)->i_private;
 152         ssize_t result = 0;
 153         int r;
 154         bool pm_pg_lock, use_bank, use_ring;
 155         unsigned instance_bank, sh_bank, se_bank, me, pipe, queue, vmid;
 156
 157         pm_pg_lock = use_bank = use_ring = false;
 158         instance_bank = sh_bank = se_bank = me = pipe = queue = vmid = 0;
 159
 160         if (size & 0x3 || *pos & 0x3 ||
 161                         ((*pos & (1ULL << 62)) && (*pos & (1ULL << 61))))
 162                 return -EINVAL;
 163
 164         /* are we reading registers for which a PG lock is necessary? */
 165         pm_pg_lock = (*pos >> 23) & 1;
 166
 167         if (*pos & (1ULL << 62)) {
 168                 se_bank = (*pos & GENMASK_ULL(33, 24)) >> 24;
 169                 sh_bank = (*pos & GENMASK_ULL(43, 34)) >> 34;
 170                 instance_bank = (*pos & GENMASK_ULL(53, 44)) >> 44;
 171
 172                 if (se_bank == 0x3FF)
 173                         se_bank = 0xFFFFFFFF;
 174                 if (sh_bank == 0x3FF)
 175                         sh_bank = 0xFFFFFFFF;
 176                 if (instance_bank == 0x3FF)
 177                         instance_bank = 0xFFFFFFFF;
 178                 use_bank = true;
 179         } else if (*pos & (1ULL << 61)) {
 180
 181                 me = (*pos & GENMASK_ULL(33, 24)) >> 24;
 182                 pipe = (*pos & GENMASK_ULL(43, 34)) >> 34;
 183                 queue = (*pos & GENMASK_ULL(53, 44)) >> 44;
 184                 vmid = (*pos & GENMASK_ULL(58, 54)) >> 54;
 185
 186                 use_ring = true;
 187         } else {
 188                 use_bank = use_ring = false;
 189         }
 190
 191         *pos &= (1UL << 22) - 1;
 192
 193         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 194         if (r < 0) {
 195                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 196                 return r;
 197         }
 198
 199         r = amdgpu_virt_enable_access_debugfs(adev);
 200         if (r < 0) {
 201                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 202                 return r;
 203         }
 204
 205         if (use_bank) {
 206                 if ((sh_bank != 0xFFFFFFFF && sh_bank >= adev->gfx.config.max_sh_per_se) ||
 207                     (se_bank != 0xFFFFFFFF && se_bank >= adev->gfx.config.max_shader_engines)) {
 208                         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 209                         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 210                         amdgpu_virt_disable_access_debugfs(adev);
 211                         return -EINVAL;
 212                 }
 213                 mutex_lock(&adev->grbm_idx_mutex);
 214                 amdgpu_gfx_select_se_sh(adev, se_bank,
 215                                         sh_bank, instance_bank);
 216         } else if (use_ring) {
 217                 mutex_lock(&adev->srbm_mutex);
 218                 amdgpu_gfx_select_me_pipe_q(adev, me, pipe, queue, vmid);
 219         }
 220
 221         if (pm_pg_lock)
 222                 mutex_lock(&adev->pm.mutex);
 223
 224         while (size) {
 225                 uint32_t value;
 226
 227                 if (read) {
 228                         value = RREG32(*pos >> 2);
 229                         r = put_user(value, (uint32_t *)buf);
 230                 } else {
 231                         r = get_user(value, (uint32_t *)buf);
 232                         if (!r)
 233                                 amdgpu_mm_wreg_mmio_rlc(adev, *pos >> 2, value);
 234                 }
 235                 if (r) {
 236                         result = r;
 237                         goto end;
 238                 }
 239
 240                 result += 4;
 241                 buf += 4;
 242                 *pos += 4;
 243                 size -= 4;
 244         }
 245
 246 end:
 247         if (use_bank) {
 248                 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
 249                 mutex_unlock(&adev->grbm_idx_mutex);
 250         } else if (use_ring) {
 251                 amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0);
 252                 mutex_unlock(&adev->srbm_mutex);
 253         }
 254
 255         if (pm_pg_lock)
 256                 mutex_unlock(&adev->pm.mutex);
 257
 258         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 259         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 260
 261         amdgpu_virt_disable_access_debugfs(adev);
 262         return result;
 263 }
 264
 265 /*
 266  * amdgpu_debugfs_regs_read - Callback for reading MMIO registers
 267  */
 268 static ssize_t amdgpu_debugfs_regs_read(struct file *f, char __user *buf,
 269                                         size_t size, loff_t *pos)
 270 {
 271         return amdgpu_debugfs_process_reg_op(true, f, buf, size, pos);
 272 }
 273
 274 /*
 275  * amdgpu_debugfs_regs_write - Callback for writing MMIO registers
 276  */
 277 static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,
 278                                          size_t size, loff_t *pos)
 279 {
 280         return amdgpu_debugfs_process_reg_op(false, f, (char __user *)buf, size, pos);
 281 }
 282
 283 static int amdgpu_debugfs_regs2_open(struct inode *inode, struct file *file)
 284 {
 285         struct amdgpu_debugfs_regs2_data *rd;
 286
 287         rd = kzalloc(sizeof *rd, GFP_KERNEL);
 288         if (!rd)
 289                 return -ENOMEM;
 290         rd->adev = file_inode(file)->i_private;
 291         file->private_data = rd;
 292         mutex_init(&rd->lock);
 293
 294         return 0;
 295 }
 296
 297 static int amdgpu_debugfs_regs2_release(struct inode *inode, struct file *file)
 298 {
 299         struct amdgpu_debugfs_regs2_data *rd = file->private_data;
 300         mutex_destroy(&rd->lock);
 301         kfree(file->private_data);
 302         return 0;
 303 }
 304
 305 static ssize_t amdgpu_debugfs_regs2_op(struct file *f, char __user *buf, u32 offset, size_t size, int write_en)
 306 {
 307         struct amdgpu_debugfs_regs2_data *rd = f->private_data;
 308         struct amdgpu_device *adev = rd->adev;
 309         ssize_t result = 0;
 310         int r;
 311         uint32_t value;
 312
 313         if (size & 0x3 || offset & 0x3)
 314                 return -EINVAL;
 315
 316         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 317         if (r < 0) {
 318                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 319                 return r;
 320         }
 321
 322         r = amdgpu_virt_enable_access_debugfs(adev);
 323         if (r < 0) {
 324                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 325                 return r;
 326         }
 327
 328         mutex_lock(&rd->lock);
 329
 330         if (rd->id.use_grbm) {
 331                 if ((rd->id.grbm.sh != 0xFFFFFFFF && rd->id.grbm.sh >= adev->gfx.config.max_sh_per_se) ||
 332                     (rd->id.grbm.se != 0xFFFFFFFF && rd->id.grbm.se >= adev->gfx.config.max_shader_engines)) {
 333                         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 334                         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 335                         amdgpu_virt_disable_access_debugfs(adev);
 336                         mutex_unlock(&rd->lock);
 337                         return -EINVAL;
 338                 }
 339                 mutex_lock(&adev->grbm_idx_mutex);
 340                 amdgpu_gfx_select_se_sh(adev, rd->id.grbm.se,
 341                                                                 rd->id.grbm.sh,
 342                                                                 rd->id.grbm.instance);
 343         }
 344
 345         if (rd->id.use_srbm) {
 346                 mutex_lock(&adev->srbm_mutex);
 347                 amdgpu_gfx_select_me_pipe_q(adev, rd->id.srbm.me, rd->id.srbm.pipe,
 348                                                                         rd->id.srbm.queue, rd->id.srbm.vmid);
 349         }
 350
 351         if (rd->id.pg_lock)
 352                 mutex_lock(&adev->pm.mutex);
 353
 354         while (size) {
 355                 if (!write_en) {
 356                         value = RREG32(offset >> 2);
 357                         r = put_user(value, (uint32_t *)buf);
 358                 } else {
 359                         r = get_user(value, (uint32_t *)buf);
 360                         if (!r)
 361                                 amdgpu_mm_wreg_mmio_rlc(adev, offset >> 2, value);
 362                 }
 363                 if (r) {
 364                         result = r;
 365                         goto end;
 366                 }
 367                 offset += 4;
 368                 size -= 4;
 369                 result += 4;
 370                 buf += 4;
 371         }
 372 end:
 373         if (rd->id.use_grbm) {
 374                 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
 375                 mutex_unlock(&adev->grbm_idx_mutex);
 376         }
 377
 378         if (rd->id.use_srbm) {
 379                 amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0);
 380                 mutex_unlock(&adev->srbm_mutex);
 381         }
 382
 383         if (rd->id.pg_lock)
 384                 mutex_unlock(&adev->pm.mutex);
 385
 386         mutex_unlock(&rd->lock);
 387
 388         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 389         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 390
 391         amdgpu_virt_disable_access_debugfs(adev);
 392         return result;
 393 }
 394
 395 static long amdgpu_debugfs_regs2_ioctl(struct file *f, unsigned int cmd, unsigned long data)
 396 {
 397         struct amdgpu_debugfs_regs2_data *rd = f->private_data;
 398         int r;
 399
 400         switch (cmd) {
 401         case AMDGPU_DEBUGFS_REGS2_IOC_SET_STATE:
 402                 mutex_lock(&rd->lock);
 403                 r = copy_from_user(&rd->id, (struct amdgpu_debugfs_regs2_iocdata *)data, sizeof rd->id);
 404                 mutex_unlock(&rd->lock);
 405                 return r ? -EINVAL : 0;
 406         default:
 407                 return -EINVAL;
 408         }
 409         return 0;
 410 }
 411
 412 static ssize_t amdgpu_debugfs_regs2_read(struct file *f, char __user *buf, size_t size, loff_t *pos)
 413 {
 414         return amdgpu_debugfs_regs2_op(f, buf, *pos, size, 0);
 415 }
 416
 417 static ssize_t amdgpu_debugfs_regs2_write(struct file *f, const char __user *buf, size_t size, loff_t *pos)
 418 {
 419         return amdgpu_debugfs_regs2_op(f, (char __user *)buf, *pos, size, 1);
 420 }
 421
 422
 423 /**
 424  * amdgpu_debugfs_regs_pcie_read - Read from a PCIE register
 425  *
 426  * @f: open file handle
 427  * @buf: User buffer to store read data in
 428  * @size: Number of bytes to read
 429  * @pos:  Offset to seek to
 430  *
 431  * The lower bits are the BYTE offset of the register to read.  This
 432  * allows reading multiple registers in a single call and having
 433  * the returned size reflect that.
 434  */
 435 static ssize_t amdgpu_debugfs_regs_pcie_read(struct file *f, char __user *buf,
 436                                         size_t size, loff_t *pos)
 437 {
 438         struct amdgpu_device *adev = file_inode(f)->i_private;
 439         ssize_t result = 0;
 440         int r;
 441
 442         if (size & 0x3 || *pos & 0x3)
 443                 return -EINVAL;
 444
 445         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 446         if (r < 0) {
 447                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 448                 return r;
 449         }
 450
 451         r = amdgpu_virt_enable_access_debugfs(adev);
 452         if (r < 0) {
 453                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 454                 return r;
 455         }
 456
 457         while (size) {
 458                 uint32_t value;
 459
 460                 value = RREG32_PCIE(*pos);
 461                 r = put_user(value, (uint32_t *)buf);
 462                 if (r) {
 463                         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 464                         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 465                         amdgpu_virt_disable_access_debugfs(adev);
 466                         return r;
 467                 }
 468
 469                 result += 4;
 470                 buf += 4;
 471                 *pos += 4;
 472                 size -= 4;
 473         }
 474
 475         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 476         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 477
 478         amdgpu_virt_disable_access_debugfs(adev);
 479         return result;
 480 }
 481
 482 /**
 483  * amdgpu_debugfs_regs_pcie_write - Write to a PCIE register
 484  *
 485  * @f: open file handle
 486  * @buf: User buffer to write data from
 487  * @size: Number of bytes to write
 488  * @pos:  Offset to seek to
 489  *
 490  * The lower bits are the BYTE offset of the register to write.  This
 491  * allows writing multiple registers in a single call and having
 492  * the returned size reflect that.
 493  */
 494 static ssize_t amdgpu_debugfs_regs_pcie_write(struct file *f, const char __user *buf,
 495                                          size_t size, loff_t *pos)
 496 {
 497         struct amdgpu_device *adev = file_inode(f)->i_private;
 498         ssize_t result = 0;
 499         int r;
 500
 501         if (size & 0x3 || *pos & 0x3)
 502                 return -EINVAL;
 503
 504         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 505         if (r < 0) {
 506                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 507                 return r;
 508         }
 509
 510         r = amdgpu_virt_enable_access_debugfs(adev);
 511         if (r < 0) {
 512                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 513                 return r;
 514         }
 515
 516         while (size) {
 517                 uint32_t value;
 518
 519                 r = get_user(value, (uint32_t *)buf);
 520                 if (r) {
 521                         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 522                         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 523                         amdgpu_virt_disable_access_debugfs(adev);
 524                         return r;
 525                 }
 526
 527                 WREG32_PCIE(*pos, value);
 528
 529                 result += 4;
 530                 buf += 4;
 531                 *pos += 4;
 532                 size -= 4;
 533         }
 534
 535         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 536         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 537
 538         amdgpu_virt_disable_access_debugfs(adev);
 539         return result;
 540 }
 541
 542 /**
 543  * amdgpu_debugfs_regs_didt_read - Read from a DIDT register
 544  *
 545  * @f: open file handle
 546  * @buf: User buffer to store read data in
 547  * @size: Number of bytes to read
 548  * @pos:  Offset to seek to
 549  *
 550  * The lower bits are the BYTE offset of the register to read.  This
 551  * allows reading multiple registers in a single call and having
 552  * the returned size reflect that.
 553  */
 554 static ssize_t amdgpu_debugfs_regs_didt_read(struct file *f, char __user *buf,
 555                                         size_t size, loff_t *pos)
 556 {
 557         struct amdgpu_device *adev = file_inode(f)->i_private;
 558         ssize_t result = 0;
 559         int r;
 560
 561         if (size & 0x3 || *pos & 0x3)
 562                 return -EINVAL;
 563
 564         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 565         if (r < 0) {
 566                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 567                 return r;
 568         }
 569
 570         r = amdgpu_virt_enable_access_debugfs(adev);
 571         if (r < 0) {
 572                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 573                 return r;
 574         }
 575
 576         while (size) {
 577                 uint32_t value;
 578
 579                 value = RREG32_DIDT(*pos >> 2);
 580                 r = put_user(value, (uint32_t *)buf);
 581                 if (r) {
 582                         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 583                         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 584                         amdgpu_virt_disable_access_debugfs(adev);
 585                         return r;
 586                 }
 587
 588                 result += 4;
 589                 buf += 4;
 590                 *pos += 4;
 591                 size -= 4;
 592         }
 593
 594         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 595         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 596
 597         amdgpu_virt_disable_access_debugfs(adev);
 598         return result;
 599 }
 600
 601 /**
 602  * amdgpu_debugfs_regs_didt_write - Write to a DIDT register
 603  *
 604  * @f: open file handle
 605  * @buf: User buffer to write data from
 606  * @size: Number of bytes to write
 607  * @pos:  Offset to seek to
 608  *
 609  * The lower bits are the BYTE offset of the register to write.  This
 610  * allows writing multiple registers in a single call and having
 611  * the returned size reflect that.
 612  */
 613 static ssize_t amdgpu_debugfs_regs_didt_write(struct file *f, const char __user *buf,
 614                                          size_t size, loff_t *pos)
 615 {
 616         struct amdgpu_device *adev = file_inode(f)->i_private;
 617         ssize_t result = 0;
 618         int r;
 619
 620         if (size & 0x3 || *pos & 0x3)
 621                 return -EINVAL;
 622
 623         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 624         if (r < 0) {
 625                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 626                 return r;
 627         }
 628
 629         r = amdgpu_virt_enable_access_debugfs(adev);
 630         if (r < 0) {
 631                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 632                 return r;
 633         }
 634
 635         while (size) {
 636                 uint32_t value;
 637
 638                 r = get_user(value, (uint32_t *)buf);
 639                 if (r) {
 640                         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 641                         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 642                         amdgpu_virt_disable_access_debugfs(adev);
 643                         return r;
 644                 }
 645
 646                 WREG32_DIDT(*pos >> 2, value);
 647
 648                 result += 4;
 649                 buf += 4;
 650                 *pos += 4;
 651                 size -= 4;
 652         }
 653
 654         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 655         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 656
 657         amdgpu_virt_disable_access_debugfs(adev);
 658         return result;
 659 }
 660
 661 /**
 662  * amdgpu_debugfs_regs_smc_read - Read from a SMC register
 663  *
 664  * @f: open file handle
 665  * @buf: User buffer to store read data in
 666  * @size: Number of bytes to read
 667  * @pos:  Offset to seek to
 668  *
 669  * The lower bits are the BYTE offset of the register to read.  This
 670  * allows reading multiple registers in a single call and having
 671  * the returned size reflect that.
 672  */
 673 static ssize_t amdgpu_debugfs_regs_smc_read(struct file *f, char __user *buf,
 674                                         size_t size, loff_t *pos)
 675 {
 676         struct amdgpu_device *adev = file_inode(f)->i_private;
 677         ssize_t result = 0;
 678         int r;
 679
 680         if (size & 0x3 || *pos & 0x3)
 681                 return -EINVAL;
 682
 683         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 684         if (r < 0) {
 685                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 686                 return r;
 687         }
 688
 689         r = amdgpu_virt_enable_access_debugfs(adev);
 690         if (r < 0) {
 691                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 692                 return r;
 693         }
 694
 695         while (size) {
 696                 uint32_t value;
 697
 698                 value = RREG32_SMC(*pos);
 699                 r = put_user(value, (uint32_t *)buf);
 700                 if (r) {
 701                         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 702                         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 703                         amdgpu_virt_disable_access_debugfs(adev);
 704                         return r;
 705                 }
 706
 707                 result += 4;
 708                 buf += 4;
 709                 *pos += 4;
 710                 size -= 4;
 711         }
 712
 713         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 714         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 715
 716         amdgpu_virt_disable_access_debugfs(adev);
 717         return result;
 718 }
 719
 720 /**
 721  * amdgpu_debugfs_regs_smc_write - Write to a SMC register
 722  *
 723  * @f: open file handle
 724  * @buf: User buffer to write data from
 725  * @size: Number of bytes to write
 726  * @pos:  Offset to seek to
 727  *
 728  * The lower bits are the BYTE offset of the register to write.  This
 729  * allows writing multiple registers in a single call and having
 730  * the returned size reflect that.
 731  */
 732 static ssize_t amdgpu_debugfs_regs_smc_write(struct file *f, const char __user *buf,
 733                                          size_t size, loff_t *pos)
 734 {
 735         struct amdgpu_device *adev = file_inode(f)->i_private;
 736         ssize_t result = 0;
 737         int r;
 738
 739         if (size & 0x3 || *pos & 0x3)
 740                 return -EINVAL;
 741
 742         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 743         if (r < 0) {
 744                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 745                 return r;
 746         }
 747
 748         r = amdgpu_virt_enable_access_debugfs(adev);
 749         if (r < 0) {
 750                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 751                 return r;
 752         }
 753
 754         while (size) {
 755                 uint32_t value;
 756
 757                 r = get_user(value, (uint32_t *)buf);
 758                 if (r) {
 759                         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 760                         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 761                         amdgpu_virt_disable_access_debugfs(adev);
 762                         return r;
 763                 }
 764
 765                 WREG32_SMC(*pos, value);
 766
 767                 result += 4;
 768                 buf += 4;
 769                 *pos += 4;
 770                 size -= 4;
 771         }
 772
 773         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 774         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 775
 776         amdgpu_virt_disable_access_debugfs(adev);
 777         return result;
 778 }
 779
 780 /**
 781  * amdgpu_debugfs_gca_config_read - Read from gfx config data
 782  *
 783  * @f: open file handle
 784  * @buf: User buffer to store read data in
 785  * @size: Number of bytes to read
 786  * @pos:  Offset to seek to
 787  *
 788  * This file is used to access configuration data in a somewhat
 789  * stable fashion.  The format is a series of DWORDs with the first
 790  * indicating which revision it is.  New content is appended to the
 791  * end so that older software can still read the data.
 792  */
 793
 794 static ssize_t amdgpu_debugfs_gca_config_read(struct file *f, char __user *buf,
 795                                         size_t size, loff_t *pos)
 796 {
 797         struct amdgpu_device *adev = file_inode(f)->i_private;
 798         ssize_t result = 0;
 799         int r;
 800         uint32_t *config, no_regs = 0;
 801
 802         if (size & 0x3 || *pos & 0x3)
 803                 return -EINVAL;
 804
 805         config = kmalloc_array(256, sizeof(*config), GFP_KERNEL);
 806         if (!config)
 807                 return -ENOMEM;
 808
 809         /* version, increment each time something is added */
 810         config[no_regs++] = 3;
 811         config[no_regs++] = adev->gfx.config.max_shader_engines;
 812         config[no_regs++] = adev->gfx.config.max_tile_pipes;
 813         config[no_regs++] = adev->gfx.config.max_cu_per_sh;
 814         config[no_regs++] = adev->gfx.config.max_sh_per_se;
 815         config[no_regs++] = adev->gfx.config.max_backends_per_se;
 816         config[no_regs++] = adev->gfx.config.max_texture_channel_caches;
 817         config[no_regs++] = adev->gfx.config.max_gprs;
 818         config[no_regs++] = adev->gfx.config.max_gs_threads;
 819         config[no_regs++] = adev->gfx.config.max_hw_contexts;
 820         config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_frontend;
 821         config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_backend;
 822         config[no_regs++] = adev->gfx.config.sc_hiz_tile_fifo_size;
 823         config[no_regs++] = adev->gfx.config.sc_earlyz_tile_fifo_size;
 824         config[no_regs++] = adev->gfx.config.num_tile_pipes;
 825         config[no_regs++] = adev->gfx.config.backend_enable_mask;
 826         config[no_regs++] = adev->gfx.config.mem_max_burst_length_bytes;
 827         config[no_regs++] = adev->gfx.config.mem_row_size_in_kb;
 828         config[no_regs++] = adev->gfx.config.shader_engine_tile_size;
 829         config[no_regs++] = adev->gfx.config.num_gpus;
 830         config[no_regs++] = adev->gfx.config.multi_gpu_tile_size;
 831         config[no_regs++] = adev->gfx.config.mc_arb_ramcfg;
 832         config[no_regs++] = adev->gfx.config.gb_addr_config;
 833         config[no_regs++] = adev->gfx.config.num_rbs;
 834
 835         /* rev==1 */
 836         config[no_regs++] = adev->rev_id;
 837         config[no_regs++] = adev->pg_flags;
 838         config[no_regs++] = adev->cg_flags;
 839
 840         /* rev==2 */
 841         config[no_regs++] = adev->family;
 842         config[no_regs++] = adev->external_rev_id;
 843
 844         /* rev==3 */
 845         config[no_regs++] = adev->pdev->device;
 846         config[no_regs++] = adev->pdev->revision;
 847         config[no_regs++] = adev->pdev->subsystem_device;
 848         config[no_regs++] = adev->pdev->subsystem_vendor;
 849
 850         while (size && (*pos < no_regs * 4)) {
 851                 uint32_t value;
 852
 853                 value = config[*pos >> 2];
 854                 r = put_user(value, (uint32_t *)buf);
 855                 if (r) {
 856                         kfree(config);
 857                         return r;
 858                 }
 859
 860                 result += 4;
 861                 buf += 4;
 862                 *pos += 4;
 863                 size -= 4;
 864         }
 865
 866         kfree(config);
 867         return result;
 868 }
 869
 870 /**
 871  * amdgpu_debugfs_sensor_read - Read from the powerplay sensors
 872  *
 873  * @f: open file handle
 874  * @buf: User buffer to store read data in
 875  * @size: Number of bytes to read
 876  * @pos:  Offset to seek to
 877  *
 878  * The offset is treated as the BYTE address of one of the sensors
 879  * enumerated in amd/include/kgd_pp_interface.h under the
 880  * 'amd_pp_sensors' enumeration.  For instance to read the UVD VCLK
 881  * you would use the offset 3 * 4 = 12.
 882  */
 883 static ssize_t amdgpu_debugfs_sensor_read(struct file *f, char __user *buf,
 884                                         size_t size, loff_t *pos)
 885 {
 886         struct amdgpu_device *adev = file_inode(f)->i_private;
 887         int idx, x, outsize, r, valuesize;
 888         uint32_t values[16];
 889
 890         if (size & 3 || *pos & 0x3)
 891                 return -EINVAL;
 892
 893         if (!adev->pm.dpm_enabled)
 894                 return -EINVAL;
 895
 896         /* convert offset to sensor number */
 897         idx = *pos >> 2;
 898
 899         valuesize = sizeof(values);
 900
 901         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 902         if (r < 0) {
 903                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 904                 return r;
 905         }
 906
 907         r = amdgpu_virt_enable_access_debugfs(adev);
 908         if (r < 0) {
 909                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 910                 return r;
 911         }
 912
 913         r = amdgpu_dpm_read_sensor(adev, idx, &values[0], &valuesize);
 914
 915         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 916         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 917
 918         if (r) {
 919                 amdgpu_virt_disable_access_debugfs(adev);
 920                 return r;
 921         }
 922
 923         if (size > valuesize) {
 924                 amdgpu_virt_disable_access_debugfs(adev);
 925                 return -EINVAL;
 926         }
 927
 928         outsize = 0;
 929         x = 0;
 930         if (!r) {
 931                 while (size) {
 932                         r = put_user(values[x++], (int32_t *)buf);
 933                         buf += 4;
 934                         size -= 4;
 935                         outsize += 4;
 936                 }
 937         }
 938
 939         amdgpu_virt_disable_access_debugfs(adev);
 940         return !r ? outsize : r;
 941 }
 942
 943 /** amdgpu_debugfs_wave_read - Read WAVE STATUS data
 944  *
 945  * @f: open file handle
 946  * @buf: User buffer to store read data in
 947  * @size: Number of bytes to read
 948  * @pos:  Offset to seek to
 949  *
 950  * The offset being sought changes which wave that the status data
 951  * will be returned for.  The bits are used as follows:
 952  *
 953  * Bits 0..6:   Byte offset into data
 954  * Bits 7..14:  SE selector
 955  * Bits 15..22: SH/SA selector
 956  * Bits 23..30: CU/{WGP+SIMD} selector
 957  * Bits 31..36: WAVE ID selector
 958  * Bits 37..44: SIMD ID selector
 959  *
 960  * The returned data begins with one DWORD of version information
 961  * Followed by WAVE STATUS registers relevant to the GFX IP version
 962  * being used.  See gfx_v8_0_read_wave_data() for an example output.
 963  */
 964 static ssize_t amdgpu_debugfs_wave_read(struct file *f, char __user *buf,
 965                                         size_t size, loff_t *pos)
 966 {
 967         struct amdgpu_device *adev = f->f_inode->i_private;
 968         int r, x;
 969         ssize_t result = 0;
 970         uint32_t offset, se, sh, cu, wave, simd, data[32];
 971
 972         if (size & 3 || *pos & 3)
 973                 return -EINVAL;
 974
 975         /* decode offset */
 976         offset = (*pos & GENMASK_ULL(6, 0));
 977         se = (*pos & GENMASK_ULL(14, 7)) >> 7;
 978         sh = (*pos & GENMASK_ULL(22, 15)) >> 15;
 979         cu = (*pos & GENMASK_ULL(30, 23)) >> 23;
 980         wave = (*pos & GENMASK_ULL(36, 31)) >> 31;
 981         simd = (*pos & GENMASK_ULL(44, 37)) >> 37;
 982
 983         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 984         if (r < 0) {
 985                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 986                 return r;
 987         }
 988
 989         r = amdgpu_virt_enable_access_debugfs(adev);
 990         if (r < 0) {
 991                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 992                 return r;
 993         }
 994
 995         /* switch to the specific se/sh/cu */
 996         mutex_lock(&adev->grbm_idx_mutex);
 997         amdgpu_gfx_select_se_sh(adev, se, sh, cu);
 998
 999         x = 0;
1000         if (adev->gfx.funcs->read_wave_data)
1001                 adev->gfx.funcs->read_wave_data(adev, simd, wave, data, &x);
1002
1003         amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF);
1004         mutex_unlock(&adev->grbm_idx_mutex);
1005
1006         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1007         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1008
1009         if (!x) {
1010                 amdgpu_virt_disable_access_debugfs(adev);
1011                 return -EINVAL;
1012         }
1013
1014         while (size && (offset < x * 4)) {
1015                 uint32_t value;
1016
1017                 value = data[offset >> 2];
1018                 r = put_user(value, (uint32_t *)buf);
1019                 if (r) {
1020                         amdgpu_virt_disable_access_debugfs(adev);
1021                         return r;
1022                 }
1023
1024                 result += 4;
1025                 buf += 4;
1026                 offset += 4;
1027                 size -= 4;
1028         }
1029
1030         amdgpu_virt_disable_access_debugfs(adev);
1031         return result;
1032 }
1033
1034 /** amdgpu_debugfs_gpr_read - Read wave gprs
1035  *
1036  * @f: open file handle
1037  * @buf: User buffer to store read data in
1038  * @size: Number of bytes to read
1039  * @pos:  Offset to seek to
1040  *
1041  * The offset being sought changes which wave that the status data
1042  * will be returned for.  The bits are used as follows:
1043  *
1044  * Bits 0..11:  Byte offset into data
1045  * Bits 12..19: SE selector
1046  * Bits 20..27: SH/SA selector
1047  * Bits 28..35: CU/{WGP+SIMD} selector
1048  * Bits 36..43: WAVE ID selector
1049  * Bits 37..44: SIMD ID selector
1050  * Bits 52..59: Thread selector
1051  * Bits 60..61: Bank selector (VGPR=0,SGPR=1)
1052  *
1053  * The return data comes from the SGPR or VGPR register bank for
1054  * the selected operational unit.
1055  */
1056 static ssize_t amdgpu_debugfs_gpr_read(struct file *f, char __user *buf,
1057                                         size_t size, loff_t *pos)
1058 {
1059         struct amdgpu_device *adev = f->f_inode->i_private;
1060         int r;
1061         ssize_t result = 0;
1062         uint32_t offset, se, sh, cu, wave, simd, thread, bank, *data;
1063
1064         if (size > 4096 || size & 3 || *pos & 3)
1065                 return -EINVAL;
1066
1067         /* decode offset */
1068         offset = (*pos & GENMASK_ULL(11, 0)) >> 2;
1069         se = (*pos & GENMASK_ULL(19, 12)) >> 12;
1070         sh = (*pos & GENMASK_ULL(27, 20)) >> 20;
1071         cu = (*pos & GENMASK_ULL(35, 28)) >> 28;
1072         wave = (*pos & GENMASK_ULL(43, 36)) >> 36;
1073         simd = (*pos & GENMASK_ULL(51, 44)) >> 44;
1074         thread = (*pos & GENMASK_ULL(59, 52)) >> 52;
1075         bank = (*pos & GENMASK_ULL(61, 60)) >> 60;
1076
1077         data = kcalloc(1024, sizeof(*data), GFP_KERNEL);
1078         if (!data)
1079                 return -ENOMEM;
1080
1081         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1082         if (r < 0)
1083                 goto err;
1084
1085         r = amdgpu_virt_enable_access_debugfs(adev);
1086         if (r < 0)
1087                 goto err;
1088
1089         /* switch to the specific se/sh/cu */
1090         mutex_lock(&adev->grbm_idx_mutex);
1091         amdgpu_gfx_select_se_sh(adev, se, sh, cu);
1092
1093         if (bank == 0) {
1094                 if (adev->gfx.funcs->read_wave_vgprs)
1095                         adev->gfx.funcs->read_wave_vgprs(adev, simd, wave, thread, offset, size>>2, data);
1096         } else {
1097                 if (adev->gfx.funcs->read_wave_sgprs)
1098                         adev->gfx.funcs->read_wave_sgprs(adev, simd, wave, offset, size>>2, data);
1099         }
1100
1101         amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF);
1102         mutex_unlock(&adev->grbm_idx_mutex);
1103
1104         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1105         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1106
1107         while (size) {
1108                 uint32_t value;
1109
1110                 value = data[result >> 2];
1111                 r = put_user(value, (uint32_t *)buf);
1112                 if (r) {
1113                         amdgpu_virt_disable_access_debugfs(adev);
1114                         goto err;
1115                 }
1116
1117                 result += 4;
1118                 buf += 4;
1119                 size -= 4;
1120         }
1121
1122         kfree(data);
1123         amdgpu_virt_disable_access_debugfs(adev);
1124         return result;
1125
1126 err:
1127         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1128         kfree(data);
1129         return r;
1130 }
1131
1132 /**
1133  * amdgpu_debugfs_gfxoff_write - Enable/disable GFXOFF
1134  *
1135  * @f: open file handle
1136  * @buf: User buffer to write data from
1137  * @size: Number of bytes to write
1138  * @pos:  Offset to seek to
1139  *
1140  * Write a 32-bit zero to disable or a 32-bit non-zero to enable
1141  */
1142 static ssize_t amdgpu_debugfs_gfxoff_write(struct file *f, const char __user *buf,
1143                                          size_t size, loff_t *pos)
1144 {
1145         struct amdgpu_device *adev = file_inode(f)->i_private;
1146         ssize_t result = 0;
1147         int r;
1148
1149         if (size & 0x3 || *pos & 0x3)
1150                 return -EINVAL;
1151
1152         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1153         if (r < 0) {
1154                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1155                 return r;
1156         }
1157
1158         while (size) {
1159                 uint32_t value;
1160
1161                 r = get_user(value, (uint32_t *)buf);
1162                 if (r) {
1163                         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1164                         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1165                         return r;
1166                 }
1167
1168                 amdgpu_gfx_off_ctrl(adev, value ? true : false);
1169
1170                 result += 4;
1171                 buf += 4;
1172                 *pos += 4;
1173                 size -= 4;
1174         }
1175
1176         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1177         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1178
1179         return result;
1180 }
1181
1182
1183 /**
1184  * amdgpu_debugfs_gfxoff_read - read gfxoff status
1185  *
1186  * @f: open file handle
1187  * @buf: User buffer to store read data in
1188  * @size: Number of bytes to read
1189  * @pos:  Offset to seek to
1190  */
1191 static ssize_t amdgpu_debugfs_gfxoff_read(struct file *f, char __user *buf,
1192                                          size_t size, loff_t *pos)
1193 {
1194         struct amdgpu_device *adev = file_inode(f)->i_private;
1195         ssize_t result = 0;
1196         int r;
1197
1198         if (size & 0x3 || *pos & 0x3)
1199                 return -EINVAL;
1200
1201         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1202         if (r < 0)
1203                 return r;
1204
1205         while (size) {
1206                 uint32_t value;
1207
1208                 r = amdgpu_get_gfx_off_status(adev, &value);
1209                 if (r) {
1210                         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1211                         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1212                         return r;
1213                 }
1214
1215                 r = put_user(value, (uint32_t *)buf);
1216                 if (r) {
1217                         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1218                         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1219                         return r;
1220                 }
1221
1222                 result += 4;
1223                 buf += 4;
1224                 *pos += 4;
1225                 size -= 4;
1226         }
1227
1228         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1229         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1230
1231         return result;
1232 }
1233
1234 static const struct file_operations amdgpu_debugfs_regs2_fops = {
1235         .owner = THIS_MODULE,
1236         .unlocked_ioctl = amdgpu_debugfs_regs2_ioctl,
1237         .read = amdgpu_debugfs_regs2_read,
1238         .write = amdgpu_debugfs_regs2_write,
1239         .open = amdgpu_debugfs_regs2_open,
1240         .release = amdgpu_debugfs_regs2_release,
1241         .llseek = default_llseek
1242 };
1243
1244 static const struct file_operations amdgpu_debugfs_regs_fops = {
1245         .owner = THIS_MODULE,
1246         .read = amdgpu_debugfs_regs_read,
1247         .write = amdgpu_debugfs_regs_write,
1248         .llseek = default_llseek
1249 };
1250 static const struct file_operations amdgpu_debugfs_regs_didt_fops = {
1251         .owner = THIS_MODULE,
1252         .read = amdgpu_debugfs_regs_didt_read,
1253         .write = amdgpu_debugfs_regs_didt_write,
1254         .llseek = default_llseek
1255 };
1256 static const struct file_operations amdgpu_debugfs_regs_pcie_fops = {
1257         .owner = THIS_MODULE,
1258         .read = amdgpu_debugfs_regs_pcie_read,
1259         .write = amdgpu_debugfs_regs_pcie_write,
1260         .llseek = default_llseek
1261 };
1262 static const struct file_operations amdgpu_debugfs_regs_smc_fops = {
1263         .owner = THIS_MODULE,
1264         .read = amdgpu_debugfs_regs_smc_read,
1265         .write = amdgpu_debugfs_regs_smc_write,
1266         .llseek = default_llseek
1267 };
1268
1269 static const struct file_operations amdgpu_debugfs_gca_config_fops = {
1270         .owner = THIS_MODULE,
1271         .read = amdgpu_debugfs_gca_config_read,
1272         .llseek = default_llseek
1273 };
1274
1275 static const struct file_operations amdgpu_debugfs_sensors_fops = {
1276         .owner = THIS_MODULE,
1277         .read = amdgpu_debugfs_sensor_read,
1278         .llseek = default_llseek
1279 };
1280
1281 static const struct file_operations amdgpu_debugfs_wave_fops = {
1282         .owner = THIS_MODULE,
1283         .read = amdgpu_debugfs_wave_read,
1284         .llseek = default_llseek
1285 };
1286 static const struct file_operations amdgpu_debugfs_gpr_fops = {
1287         .owner = THIS_MODULE,
1288         .read = amdgpu_debugfs_gpr_read,
1289         .llseek = default_llseek
1290 };
1291
1292 static const struct file_operations amdgpu_debugfs_gfxoff_fops = {
1293         .owner = THIS_MODULE,
1294         .read = amdgpu_debugfs_gfxoff_read,
1295         .write = amdgpu_debugfs_gfxoff_write,
1296         .llseek = default_llseek
1297 };
1298
1299 static const struct file_operations *debugfs_regs[] = {
1300         &amdgpu_debugfs_regs_fops,
1301         &amdgpu_debugfs_regs2_fops,
1302         &amdgpu_debugfs_regs_didt_fops,
1303         &amdgpu_debugfs_regs_pcie_fops,
1304         &amdgpu_debugfs_regs_smc_fops,
1305         &amdgpu_debugfs_gca_config_fops,
1306         &amdgpu_debugfs_sensors_fops,
1307         &amdgpu_debugfs_wave_fops,
1308         &amdgpu_debugfs_gpr_fops,
1309         &amdgpu_debugfs_gfxoff_fops,
1310 };
1311
1312 static const char *debugfs_regs_names[] = {
1313         "amdgpu_regs",
1314         "amdgpu_regs2",
1315         "amdgpu_regs_didt",
1316         "amdgpu_regs_pcie",
1317         "amdgpu_regs_smc",
1318         "amdgpu_gca_config",
1319         "amdgpu_sensors",
1320         "amdgpu_wave",
1321         "amdgpu_gpr",
1322         "amdgpu_gfxoff",
1323 };
1324
1325 /**
1326  * amdgpu_debugfs_regs_init -   Initialize debugfs entries that provide
1327  *                              register access.
1328  *
1329  * @adev: The device to attach the debugfs entries to
1330  */
1331 int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
1332 {
1333         struct drm_minor *minor = adev_to_drm(adev)->primary;
1334         struct dentry *ent, *root = minor->debugfs_root;
1335         unsigned int i;
1336
1337         for (i = 0; i < ARRAY_SIZE(debugfs_regs); i++) {
1338                 ent = debugfs_create_file(debugfs_regs_names[i],
1339                                           S_IFREG | S_IRUGO, root,
1340                                           adev, debugfs_regs[i]);
1341                 if (!i && !IS_ERR_OR_NULL(ent))
1342                         i_size_write(ent->d_inode, adev->rmmio_size);
1343         }
1344
1345         return 0;
1346 }
1347
1348 static int amdgpu_debugfs_test_ib_show(struct seq_file *m, void *unused)
1349 {
1350         struct amdgpu_device *adev = (struct amdgpu_device *)m->private;
1351         struct drm_device *dev = adev_to_drm(adev);
1352         int r = 0, i;
1353
1354         r = pm_runtime_get_sync(dev->dev);
1355         if (r < 0) {
1356                 pm_runtime_put_autosuspend(dev->dev);
1357                 return r;
1358         }
1359
1360         /* Avoid accidently unparking the sched thread during GPU reset */
1361         r = down_read_killable(&adev->reset_sem);
1362         if (r)
1363                 return r;
1364
1365         /* hold on the scheduler */
1366         for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
1367                 struct amdgpu_ring *ring = adev->rings[i];
1368
1369                 if (!ring || !ring->sched.thread)
1370                         continue;
1371                 kthread_park(ring->sched.thread);
1372         }
1373
1374         seq_printf(m, "run ib test:\n");
1375         r = amdgpu_ib_ring_tests(adev);
1376         if (r)
1377                 seq_printf(m, "ib ring tests failed (%d).\n", r);
1378         else
1379                 seq_printf(m, "ib ring tests passed.\n");
1380
1381         /* go on the scheduler */
1382         for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
1383                 struct amdgpu_ring *ring = adev->rings[i];
1384
1385                 if (!ring || !ring->sched.thread)
1386                         continue;
1387                 kthread_unpark(ring->sched.thread);
1388         }
1389
1390         up_read(&adev->reset_sem);
1391
1392         pm_runtime_mark_last_busy(dev->dev);
1393         pm_runtime_put_autosuspend(dev->dev);
1394
1395         return 0;
1396 }
1397
1398 static int amdgpu_debugfs_evict_vram(void *data, u64 *val)
1399 {
1400         struct amdgpu_device *adev = (struct amdgpu_device *)data;
1401         struct drm_device *dev = adev_to_drm(adev);
1402         int r;
1403
1404         r = pm_runtime_get_sync(dev->dev);
1405         if (r < 0) {
1406                 pm_runtime_put_autosuspend(dev->dev);
1407                 return r;
1408         }
1409
1410         *val = amdgpu_bo_evict_vram(adev);
1411
1412         pm_runtime_mark_last_busy(dev->dev);
1413         pm_runtime_put_autosuspend(dev->dev);
1414
1415         return 0;
1416 }
1417
1418
1419 static int amdgpu_debugfs_evict_gtt(void *data, u64 *val)
1420 {
1421         struct amdgpu_device *adev = (struct amdgpu_device *)data;
1422         struct drm_device *dev = adev_to_drm(adev);
1423         struct ttm_resource_manager *man;
1424         int r;
1425
1426         r = pm_runtime_get_sync(dev->dev);
1427         if (r < 0) {
1428                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1429                 return r;
1430         }
1431
1432         man = ttm_manager_type(&adev->mman.bdev, TTM_PL_TT);
1433         *val = ttm_resource_manager_evict_all(&adev->mman.bdev, man);
1434
1435         pm_runtime_mark_last_busy(dev->dev);
1436         pm_runtime_put_autosuspend(dev->dev);
1437
1438         return 0;
1439 }
1440
1441
1442 static int amdgpu_debugfs_vm_info_show(struct seq_file *m, void *unused)
1443 {
1444         struct amdgpu_device *adev = (struct amdgpu_device *)m->private;
1445         struct drm_device *dev = adev_to_drm(adev);
1446         struct drm_file *file;
1447         int r;
1448
1449         r = mutex_lock_interruptible(&dev->filelist_mutex);
1450         if (r)
1451                 return r;
1452
1453         list_for_each_entry(file, &dev->filelist, lhead) {
1454                 struct amdgpu_fpriv *fpriv = file->driver_priv;
1455                 struct amdgpu_vm *vm = &fpriv->vm;
1456
1457                 seq_printf(m, "pid:%d\tProcess:%s ----------\n",
1458                                 vm->task_info.pid, vm->task_info.process_name);
1459                 r = amdgpu_bo_reserve(vm->root.bo, true);
1460                 if (r)
1461                         break;
1462                 amdgpu_debugfs_vm_bo_info(vm, m);
1463                 amdgpu_bo_unreserve(vm->root.bo);
1464         }
1465
1466         mutex_unlock(&dev->filelist_mutex);
1467
1468         return r;
1469 }
1470
1471 DEFINE_SHOW_ATTRIBUTE(amdgpu_debugfs_test_ib);
1472 DEFINE_SHOW_ATTRIBUTE(amdgpu_debugfs_vm_info);
1473 DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_evict_vram_fops, amdgpu_debugfs_evict_vram,
1474                          NULL, "%lld\n");
1475 DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_evict_gtt_fops, amdgpu_debugfs_evict_gtt,
1476                          NULL, "%lld\n");
1477
1478 static void amdgpu_ib_preempt_fences_swap(struct amdgpu_ring *ring,
1479                                           struct dma_fence **fences)
1480 {
1481         struct amdgpu_fence_driver *drv = &ring->fence_drv;
1482         uint32_t sync_seq, last_seq;
1483
1484         last_seq = atomic_read(&ring->fence_drv.last_seq);
1485         sync_seq = ring->fence_drv.sync_seq;
1486
1487         last_seq &= drv->num_fences_mask;
1488         sync_seq &= drv->num_fences_mask;
1489
1490         do {
1491                 struct dma_fence *fence, **ptr;
1492
1493                 ++last_seq;
1494                 last_seq &= drv->num_fences_mask;
1495                 ptr = &drv->fences[last_seq];
1496
1497                 fence = rcu_dereference_protected(*ptr, 1);
1498                 RCU_INIT_POINTER(*ptr, NULL);
1499
1500                 if (!fence)
1501                         continue;
1502
1503                 fences[last_seq] = fence;
1504
1505         } while (last_seq != sync_seq);
1506 }
1507
1508 static void amdgpu_ib_preempt_signal_fences(struct dma_fence **fences,
1509                                             int length)
1510 {
1511         int i;
1512         struct dma_fence *fence;
1513
1514         for (i = 0; i < length; i++) {
1515                 fence = fences[i];
1516                 if (!fence)
1517                         continue;
1518                 dma_fence_signal(fence);
1519                 dma_fence_put(fence);
1520         }
1521 }
1522
1523 static void amdgpu_ib_preempt_job_recovery(struct drm_gpu_scheduler *sched)
1524 {
1525         struct drm_sched_job *s_job;
1526         struct dma_fence *fence;
1527
1528         spin_lock(&sched->job_list_lock);
1529         list_for_each_entry(s_job, &sched->pending_list, list) {
1530                 fence = sched->ops->run_job(s_job);
1531                 dma_fence_put(fence);
1532         }
1533         spin_unlock(&sched->job_list_lock);
1534 }
1535
1536 static void amdgpu_ib_preempt_mark_partial_job(struct amdgpu_ring *ring)
1537 {
1538         struct amdgpu_job *job;
1539         struct drm_sched_job *s_job, *tmp;
1540         uint32_t preempt_seq;
1541         struct dma_fence *fence, **ptr;
1542         struct amdgpu_fence_driver *drv = &ring->fence_drv;
1543         struct drm_gpu_scheduler *sched = &ring->sched;
1544         bool preempted = true;
1545
1546         if (ring->funcs->type != AMDGPU_RING_TYPE_GFX)
1547                 return;
1548
1549         preempt_seq = le32_to_cpu(*(drv->cpu_addr + 2));
1550         if (preempt_seq <= atomic_read(&drv->last_seq)) {
1551                 preempted = false;
1552                 goto no_preempt;
1553         }
1554
1555         preempt_seq &= drv->num_fences_mask;
1556         ptr = &drv->fences[preempt_seq];
1557         fence = rcu_dereference_protected(*ptr, 1);
1558
1559 no_preempt:
1560         spin_lock(&sched->job_list_lock);
1561         list_for_each_entry_safe(s_job, tmp, &sched->pending_list, list) {
1562                 if (dma_fence_is_signaled(&s_job->s_fence->finished)) {
1563                         /* remove job from ring_mirror_list */
1564                         list_del_init(&s_job->list);
1565                         sched->ops->free_job(s_job);
1566                         continue;
1567                 }
1568                 job = to_amdgpu_job(s_job);
1569                 if (preempted && (&job->hw_fence) == fence)
1570                         /* mark the job as preempted */
1571                         job->preemption_status |= AMDGPU_IB_PREEMPTED;
1572         }
1573         spin_unlock(&sched->job_list_lock);
1574 }
1575
1576 static int amdgpu_debugfs_ib_preempt(void *data, u64 val)
1577 {
1578         int r, resched, length;
1579         struct amdgpu_ring *ring;
1580         struct dma_fence **fences = NULL;
1581         struct amdgpu_device *adev = (struct amdgpu_device *)data;
1582
1583         if (val >= AMDGPU_MAX_RINGS)
1584                 return -EINVAL;
1585
1586         ring = adev->rings[val];
1587
1588         if (!ring || !ring->funcs->preempt_ib || !ring->sched.thread)
1589                 return -EINVAL;
1590
1591         /* the last preemption failed */
1592         if (ring->trail_seq != le32_to_cpu(*ring->trail_fence_cpu_addr))
1593                 return -EBUSY;
1594
1595         length = ring->fence_drv.num_fences_mask + 1;
1596         fences = kcalloc(length, sizeof(void *), GFP_KERNEL);
1597         if (!fences)
1598                 return -ENOMEM;
1599
1600         /* Avoid accidently unparking the sched thread during GPU reset */
1601         r = down_read_killable(&adev->reset_sem);
1602         if (r)
1603                 goto pro_end;
1604
1605         /* stop the scheduler */
1606         kthread_park(ring->sched.thread);
1607
1608         resched = ttm_bo_lock_delayed_workqueue(&adev->mman.bdev);
1609
1610         /* preempt the IB */
1611         r = amdgpu_ring_preempt_ib(ring);
1612         if (r) {
1613                 DRM_WARN("failed to preempt ring %d\n", ring->idx);
1614                 goto failure;
1615         }
1616
1617         amdgpu_fence_process(ring);
1618
1619         if (atomic_read(&ring->fence_drv.last_seq) !=
1620             ring->fence_drv.sync_seq) {
1621                 DRM_INFO("ring %d was preempted\n", ring->idx);
1622
1623                 amdgpu_ib_preempt_mark_partial_job(ring);
1624
1625                 /* swap out the old fences */
1626                 amdgpu_ib_preempt_fences_swap(ring, fences);
1627
1628                 amdgpu_fence_driver_force_completion(ring);
1629
1630                 /* resubmit unfinished jobs */
1631                 amdgpu_ib_preempt_job_recovery(&ring->sched);
1632
1633                 /* wait for jobs finished */
1634                 amdgpu_fence_wait_empty(ring);
1635
1636                 /* signal the old fences */
1637                 amdgpu_ib_preempt_signal_fences(fences, length);
1638         }
1639
1640 failure:
1641         /* restart the scheduler */
1642         kthread_unpark(ring->sched.thread);
1643
1644         up_read(&adev->reset_sem);
1645
1646         ttm_bo_unlock_delayed_workqueue(&adev->mman.bdev, resched);
1647
1648 pro_end:
1649         kfree(fences);
1650
1651         return r;
1652 }
1653
1654 static int amdgpu_debugfs_sclk_set(void *data, u64 val)
1655 {
1656         int ret = 0;
1657         uint32_t max_freq, min_freq;
1658         struct amdgpu_device *adev = (struct amdgpu_device *)data;
1659
1660         if (amdgpu_sriov_vf(adev) && !amdgpu_sriov_is_pp_one_vf(adev))
1661                 return -EINVAL;
1662
1663         ret = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1664         if (ret < 0) {
1665                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1666                 return ret;
1667         }
1668
1669         if (is_support_sw_smu(adev)) {
1670                 ret = smu_get_dpm_freq_range(&adev->smu, SMU_SCLK, &min_freq, &max_freq);
1671                 if (ret || val > max_freq || val < min_freq)
1672                         return -EINVAL;
1673                 ret = smu_set_soft_freq_range(&adev->smu, SMU_SCLK, (uint32_t)val, (uint32_t)val);
1674         } else {
1675                 return 0;
1676         }
1677
1678         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1679         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1680
1681         if (ret)
1682                 return -EINVAL;
1683
1684         return 0;
1685 }
1686
1687 DEFINE_DEBUGFS_ATTRIBUTE(fops_ib_preempt, NULL,
1688                         amdgpu_debugfs_ib_preempt, "%llu\n");
1689
1690 DEFINE_DEBUGFS_ATTRIBUTE(fops_sclk_set, NULL,
1691                         amdgpu_debugfs_sclk_set, "%llu\n");
1692
1693 int amdgpu_debugfs_init(struct amdgpu_device *adev)
1694 {
1695         struct dentry *root = adev_to_drm(adev)->primary->debugfs_root;
1696         struct dentry *ent;
1697         int r, i;
1698
1699
1700
1701         ent = debugfs_create_file("amdgpu_preempt_ib", 0600, root, adev,
1702                                   &fops_ib_preempt);
1703         if (!ent) {
1704                 DRM_ERROR("unable to create amdgpu_preempt_ib debugsfs file\n");
1705                 return -EIO;
1706         }
1707
1708         ent = debugfs_create_file("amdgpu_force_sclk", 0200, root, adev,
1709                                   &fops_sclk_set);
1710         if (!ent) {
1711                 DRM_ERROR("unable to create amdgpu_set_sclk debugsfs file\n");
1712                 return -EIO;
1713         }
1714
1715         /* Register debugfs entries for amdgpu_ttm */
1716         amdgpu_ttm_debugfs_init(adev);
1717         amdgpu_debugfs_pm_init(adev);
1718         amdgpu_debugfs_sa_init(adev);
1719         amdgpu_debugfs_fence_init(adev);
1720         amdgpu_debugfs_gem_init(adev);
1721
1722         r = amdgpu_debugfs_regs_init(adev);
1723         if (r)
1724                 DRM_ERROR("registering register debugfs failed (%d).\n", r);
1725
1726         amdgpu_debugfs_firmware_init(adev);
1727
1728 #if defined(CONFIG_DRM_AMD_DC)
1729         if (amdgpu_device_has_dc_support(adev))
1730                 dtn_debugfs_init(adev);
1731 #endif
1732
1733         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
1734                 struct amdgpu_ring *ring = adev->rings[i];
1735
1736                 if (!ring)
1737                         continue;
1738
1739                 if (amdgpu_debugfs_ring_init(adev, ring)) {
1740                         DRM_ERROR("Failed to register debugfs file for rings !\n");
1741                 }
1742         }
1743
1744         amdgpu_ras_debugfs_create_all(adev);
1745         amdgpu_debugfs_autodump_init(adev);
1746         amdgpu_rap_debugfs_init(adev);
1747         amdgpu_securedisplay_debugfs_init(adev);
1748         amdgpu_fw_attestation_debugfs_init(adev);
1749
1750         debugfs_create_file("amdgpu_evict_vram", 0444, root, adev,
1751                             &amdgpu_evict_vram_fops);
1752         debugfs_create_file("amdgpu_evict_gtt", 0444, root, adev,
1753                             &amdgpu_evict_gtt_fops);
1754         debugfs_create_file("amdgpu_test_ib", 0444, root, adev,
1755                             &amdgpu_debugfs_test_ib_fops);
1756         debugfs_create_file("amdgpu_vm_info", 0444, root, adev,
1757                             &amdgpu_debugfs_vm_info_fops);
1758
1759         adev->debugfs_vbios_blob.data = adev->bios;
1760         adev->debugfs_vbios_blob.size = adev->bios_size;
1761         debugfs_create_blob("amdgpu_vbios", 0444, root,
1762                             &adev->debugfs_vbios_blob);
1763
1764         return 0;
1765 }
1766
1767 #else
1768 int amdgpu_debugfs_init(struct amdgpu_device *adev)
1769 {
1770         return 0;
1771 }
1772 int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
1773 {
1774         return 0;
1775 }
1776 #endif