drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c

   1 /*
   2  * Copyright 2008 Advanced Micro Devices, Inc.
   3  * Copyright 2008 Red Hat Inc.
   4  * Copyright 2009 Jerome Glisse.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22  * OTHER DEALINGS IN THE SOFTWARE.
  23  *
  24  */
  25
  26 #include <linux/kthread.h>
  27 #include <linux/pci.h>
  28 #include <linux/uaccess.h>
  29 #include <linux/pm_runtime.h>
  30
  31 #include "amdgpu.h"
  32 #include "amdgpu_pm.h"
  33 #include "amdgpu_dm_debugfs.h"
  34 #include "amdgpu_ras.h"
  35 #include "amdgpu_rap.h"
  36 #include "amdgpu_securedisplay.h"
  37 #include "amdgpu_fw_attestation.h"
  38 #include "amdgpu_umr.h"
  39
  40 #include "amdgpu_reset.h"
  41 #include "amdgpu_psp_ta.h"
  42
  43 #if defined(CONFIG_DEBUG_FS)
  44
  45 /**
  46  * amdgpu_debugfs_process_reg_op - Handle MMIO register reads/writes
  47  *
  48  * @read: True if reading
  49  * @f: open file handle
  50  * @buf: User buffer to write/read to
  51  * @size: Number of bytes to write/read
  52  * @pos:  Offset to seek to
  53  *
  54  * This debugfs entry has special meaning on the offset being sought.
  55  * Various bits have different meanings:
  56  *
  57  * Bit 62:  Indicates a GRBM bank switch is needed
  58  * Bit 61:  Indicates a SRBM bank switch is needed (implies bit 62 is
  59  *          zero)
  60  * Bits 24..33: The SE or ME selector if needed
  61  * Bits 34..43: The SH (or SA) or PIPE selector if needed
  62  * Bits 44..53: The INSTANCE (or CU/WGP) or QUEUE selector if needed
  63  *
  64  * Bit 23:  Indicates that the PM power gating lock should be held
  65  *          This is necessary to read registers that might be
  66  *          unreliable during a power gating transistion.
  67  *
  68  * The lower bits are the BYTE offset of the register to read.  This
  69  * allows reading multiple registers in a single call and having
  70  * the returned size reflect that.
  71  */
  72 static int  amdgpu_debugfs_process_reg_op(bool read, struct file *f,
  73                 char __user *buf, size_t size, loff_t *pos)
  74 {
  75         struct amdgpu_device *adev = file_inode(f)->i_private;
  76         ssize_t result = 0;
  77         int r;
  78         bool pm_pg_lock, use_bank, use_ring;
  79         unsigned int instance_bank, sh_bank, se_bank, me, pipe, queue, vmid;
  80
  81         pm_pg_lock = use_bank = use_ring = false;
  82         instance_bank = sh_bank = se_bank = me = pipe = queue = vmid = 0;
  83
  84         if (size & 0x3 || *pos & 0x3 ||
  85                         ((*pos & (1ULL << 62)) && (*pos & (1ULL << 61))))
  86                 return -EINVAL;
  87
  88         /* are we reading registers for which a PG lock is necessary? */
  89         pm_pg_lock = (*pos >> 23) & 1;
  90
  91         if (*pos & (1ULL << 62)) {
  92                 se_bank = (*pos & GENMASK_ULL(33, 24)) >> 24;
  93                 sh_bank = (*pos & GENMASK_ULL(43, 34)) >> 34;
  94                 instance_bank = (*pos & GENMASK_ULL(53, 44)) >> 44;
  95
  96                 if (se_bank == 0x3FF)
  97                         se_bank = 0xFFFFFFFF;
  98                 if (sh_bank == 0x3FF)
  99                         sh_bank = 0xFFFFFFFF;
 100                 if (instance_bank == 0x3FF)
 101                         instance_bank = 0xFFFFFFFF;
 102                 use_bank = true;
 103         } else if (*pos & (1ULL << 61)) {
 104
 105                 me = (*pos & GENMASK_ULL(33, 24)) >> 24;
 106                 pipe = (*pos & GENMASK_ULL(43, 34)) >> 34;
 107                 queue = (*pos & GENMASK_ULL(53, 44)) >> 44;
 108                 vmid = (*pos & GENMASK_ULL(58, 54)) >> 54;
 109
 110                 use_ring = true;
 111         } else {
 112                 use_bank = use_ring = false;
 113         }
 114
 115         *pos &= (1UL << 22) - 1;
 116
 117         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 118         if (r < 0) {
 119                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 120                 return r;
 121         }
 122
 123         r = amdgpu_virt_enable_access_debugfs(adev);
 124         if (r < 0) {
 125                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 126                 return r;
 127         }
 128
 129         if (use_bank) {
 130                 if ((sh_bank != 0xFFFFFFFF && sh_bank >= adev->gfx.config.max_sh_per_se) ||
 131                     (se_bank != 0xFFFFFFFF && se_bank >= adev->gfx.config.max_shader_engines)) {
 132                         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 133                         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 134                         amdgpu_virt_disable_access_debugfs(adev);
 135                         return -EINVAL;
 136                 }
 137                 mutex_lock(&adev->grbm_idx_mutex);
 138                 amdgpu_gfx_select_se_sh(adev, se_bank,
 139                                         sh_bank, instance_bank, 0);
 140         } else if (use_ring) {
 141                 mutex_lock(&adev->srbm_mutex);
 142                 amdgpu_gfx_select_me_pipe_q(adev, me, pipe, queue, vmid, 0);
 143         }
 144
 145         if (pm_pg_lock)
 146                 mutex_lock(&adev->pm.mutex);
 147
 148         while (size) {
 149                 uint32_t value;
 150
 151                 if (read) {
 152                         value = RREG32(*pos >> 2);
 153                         r = put_user(value, (uint32_t *)buf);
 154                 } else {
 155                         r = get_user(value, (uint32_t *)buf);
 156                         if (!r)
 157                                 amdgpu_mm_wreg_mmio_rlc(adev, *pos >> 2, value);
 158                 }
 159                 if (r) {
 160                         result = r;
 161                         goto end;
 162                 }
 163
 164                 result += 4;
 165                 buf += 4;
 166                 *pos += 4;
 167                 size -= 4;
 168         }
 169
 170 end:
 171         if (use_bank) {
 172                 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
 173                 mutex_unlock(&adev->grbm_idx_mutex);
 174         } else if (use_ring) {
 175                 amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0);
 176                 mutex_unlock(&adev->srbm_mutex);
 177         }
 178
 179         if (pm_pg_lock)
 180                 mutex_unlock(&adev->pm.mutex);
 181
 182         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 183         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 184
 185         amdgpu_virt_disable_access_debugfs(adev);
 186         return result;
 187 }
 188
 189 /*
 190  * amdgpu_debugfs_regs_read - Callback for reading MMIO registers
 191  */
 192 static ssize_t amdgpu_debugfs_regs_read(struct file *f, char __user *buf,
 193                                         size_t size, loff_t *pos)
 194 {
 195         return amdgpu_debugfs_process_reg_op(true, f, buf, size, pos);
 196 }
 197
 198 /*
 199  * amdgpu_debugfs_regs_write - Callback for writing MMIO registers
 200  */
 201 static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,
 202                                          size_t size, loff_t *pos)
 203 {
 204         return amdgpu_debugfs_process_reg_op(false, f, (char __user *)buf, size, pos);
 205 }
 206
 207 static int amdgpu_debugfs_regs2_open(struct inode *inode, struct file *file)
 208 {
 209         struct amdgpu_debugfs_regs2_data *rd;
 210
 211         rd = kzalloc(sizeof(*rd), GFP_KERNEL);
 212         if (!rd)
 213                 return -ENOMEM;
 214         rd->adev = file_inode(file)->i_private;
 215         file->private_data = rd;
 216         mutex_init(&rd->lock);
 217
 218         return 0;
 219 }
 220
 221 static int amdgpu_debugfs_regs2_release(struct inode *inode, struct file *file)
 222 {
 223         struct amdgpu_debugfs_regs2_data *rd = file->private_data;
 224
 225         mutex_destroy(&rd->lock);
 226         kfree(file->private_data);
 227         return 0;
 228 }
 229
 230 static ssize_t amdgpu_debugfs_regs2_op(struct file *f, char __user *buf, u32 offset, size_t size, int write_en)
 231 {
 232         struct amdgpu_debugfs_regs2_data *rd = f->private_data;
 233         struct amdgpu_device *adev = rd->adev;
 234         ssize_t result = 0;
 235         int r;
 236         uint32_t value;
 237
 238         if (size & 0x3 || offset & 0x3)
 239                 return -EINVAL;
 240
 241         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 242         if (r < 0) {
 243                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 244                 return r;
 245         }
 246
 247         r = amdgpu_virt_enable_access_debugfs(adev);
 248         if (r < 0) {
 249                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 250                 return r;
 251         }
 252
 253         mutex_lock(&rd->lock);
 254
 255         if (rd->id.use_grbm) {
 256                 if ((rd->id.grbm.sh != 0xFFFFFFFF && rd->id.grbm.sh >= adev->gfx.config.max_sh_per_se) ||
 257                     (rd->id.grbm.se != 0xFFFFFFFF && rd->id.grbm.se >= adev->gfx.config.max_shader_engines)) {
 258                         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 259                         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 260                         amdgpu_virt_disable_access_debugfs(adev);
 261                         mutex_unlock(&rd->lock);
 262                         return -EINVAL;
 263                 }
 264                 mutex_lock(&adev->grbm_idx_mutex);
 265                 amdgpu_gfx_select_se_sh(adev, rd->id.grbm.se,
 266                                                   rd->id.grbm.sh,
 267                                                   rd->id.grbm.instance, rd->id.xcc_id);
 268         }
 269
 270         if (rd->id.use_srbm) {
 271                 mutex_lock(&adev->srbm_mutex);
 272                 amdgpu_gfx_select_me_pipe_q(adev, rd->id.srbm.me, rd->id.srbm.pipe,
 273                                             rd->id.srbm.queue, rd->id.srbm.vmid, rd->id.xcc_id);
 274         }
 275
 276         if (rd->id.pg_lock)
 277                 mutex_lock(&adev->pm.mutex);
 278
 279         while (size) {
 280                 if (!write_en) {
 281                         value = RREG32(offset >> 2);
 282                         r = put_user(value, (uint32_t *)buf);
 283                 } else {
 284                         r = get_user(value, (uint32_t *)buf);
 285                         if (!r)
 286                                 amdgpu_mm_wreg_mmio_rlc(adev, offset >> 2, value);
 287                 }
 288                 if (r) {
 289                         result = r;
 290                         goto end;
 291                 }
 292                 offset += 4;
 293                 size -= 4;
 294                 result += 4;
 295                 buf += 4;
 296         }
 297 end:
 298         if (rd->id.use_grbm) {
 299                 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, rd->id.xcc_id);
 300                 mutex_unlock(&adev->grbm_idx_mutex);
 301         }
 302
 303         if (rd->id.use_srbm) {
 304                 amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, rd->id.xcc_id);
 305                 mutex_unlock(&adev->srbm_mutex);
 306         }
 307
 308         if (rd->id.pg_lock)
 309                 mutex_unlock(&adev->pm.mutex);
 310
 311         mutex_unlock(&rd->lock);
 312
 313         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 314         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 315
 316         amdgpu_virt_disable_access_debugfs(adev);
 317         return result;
 318 }
 319
 320 static long amdgpu_debugfs_regs2_ioctl(struct file *f, unsigned int cmd, unsigned long data)
 321 {
 322         struct amdgpu_debugfs_regs2_data *rd = f->private_data;
 323         struct amdgpu_debugfs_regs2_iocdata v1_data;
 324         int r;
 325
 326         mutex_lock(&rd->lock);
 327
 328         switch (cmd) {
 329         case AMDGPU_DEBUGFS_REGS2_IOC_SET_STATE_V2:
 330                 r = copy_from_user(&rd->id, (struct amdgpu_debugfs_regs2_iocdata_v2 *)data,
 331                                    sizeof(rd->id));
 332                 if (r)
 333                         r = -EINVAL;
 334                 goto done;
 335         case AMDGPU_DEBUGFS_REGS2_IOC_SET_STATE:
 336                 r = copy_from_user(&v1_data, (struct amdgpu_debugfs_regs2_iocdata *)data,
 337                                    sizeof(v1_data));
 338                 if (r) {
 339                         r = -EINVAL;
 340                         goto done;
 341                 }
 342                 goto v1_copy;
 343         default:
 344                 r = -EINVAL;
 345                 goto done;
 346         }
 347
 348 v1_copy:
 349         rd->id.use_srbm = v1_data.use_srbm;
 350         rd->id.use_grbm = v1_data.use_grbm;
 351         rd->id.pg_lock = v1_data.pg_lock;
 352         rd->id.grbm.se = v1_data.grbm.se;
 353         rd->id.grbm.sh = v1_data.grbm.sh;
 354         rd->id.grbm.instance = v1_data.grbm.instance;
 355         rd->id.srbm.me = v1_data.srbm.me;
 356         rd->id.srbm.pipe = v1_data.srbm.pipe;
 357         rd->id.srbm.queue = v1_data.srbm.queue;
 358         rd->id.xcc_id = 0;
 359 done:
 360         mutex_unlock(&rd->lock);
 361         return r;
 362 }
 363
 364 static ssize_t amdgpu_debugfs_regs2_read(struct file *f, char __user *buf, size_t size, loff_t *pos)
 365 {
 366         return amdgpu_debugfs_regs2_op(f, buf, *pos, size, 0);
 367 }
 368
 369 static ssize_t amdgpu_debugfs_regs2_write(struct file *f, const char __user *buf, size_t size, loff_t *pos)
 370 {
 371         return amdgpu_debugfs_regs2_op(f, (char __user *)buf, *pos, size, 1);
 372 }
 373
 374 static int amdgpu_debugfs_gprwave_open(struct inode *inode, struct file *file)
 375 {
 376         struct amdgpu_debugfs_gprwave_data *rd;
 377
 378         rd = kzalloc(sizeof *rd, GFP_KERNEL);
 379         if (!rd)
 380                 return -ENOMEM;
 381         rd->adev = file_inode(file)->i_private;
 382         file->private_data = rd;
 383         mutex_init(&rd->lock);
 384
 385         return 0;
 386 }
 387
 388 static int amdgpu_debugfs_gprwave_release(struct inode *inode, struct file *file)
 389 {
 390         struct amdgpu_debugfs_gprwave_data *rd = file->private_data;
 391         mutex_destroy(&rd->lock);
 392         kfree(file->private_data);
 393         return 0;
 394 }
 395
 396 static ssize_t amdgpu_debugfs_gprwave_read(struct file *f, char __user *buf, size_t size, loff_t *pos)
 397 {
 398         struct amdgpu_debugfs_gprwave_data *rd = f->private_data;
 399         struct amdgpu_device *adev = rd->adev;
 400         ssize_t result = 0;
 401         int r;
 402         uint32_t *data, x;
 403
 404         if (size & 0x3 || *pos & 0x3)
 405                 return -EINVAL;
 406
 407         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 408         if (r < 0) {
 409                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 410                 return r;
 411         }
 412
 413         r = amdgpu_virt_enable_access_debugfs(adev);
 414         if (r < 0) {
 415                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 416                 return r;
 417         }
 418
 419         data = kcalloc(1024, sizeof(*data), GFP_KERNEL);
 420         if (!data) {
 421                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 422                 amdgpu_virt_disable_access_debugfs(adev);
 423                 return -ENOMEM;
 424         }
 425
 426         /* switch to the specific se/sh/cu */
 427         mutex_lock(&adev->grbm_idx_mutex);
 428         amdgpu_gfx_select_se_sh(adev, rd->id.se, rd->id.sh, rd->id.cu, rd->id.xcc_id);
 429
 430         if (!rd->id.gpr_or_wave) {
 431                 x = 0;
 432                 if (adev->gfx.funcs->read_wave_data)
 433                         adev->gfx.funcs->read_wave_data(adev, rd->id.xcc_id, rd->id.simd, rd->id.wave, data, &x);
 434         } else {
 435                 x = size >> 2;
 436                 if (rd->id.gpr.vpgr_or_sgpr) {
 437                         if (adev->gfx.funcs->read_wave_vgprs)
 438                                 adev->gfx.funcs->read_wave_vgprs(adev, rd->id.xcc_id, rd->id.simd, rd->id.wave, rd->id.gpr.thread, *pos, size>>2, data);
 439                 } else {
 440                         if (adev->gfx.funcs->read_wave_sgprs)
 441                                 adev->gfx.funcs->read_wave_sgprs(adev, rd->id.xcc_id, rd->id.simd, rd->id.wave, *pos, size>>2, data);
 442                 }
 443         }
 444
 445         amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, rd->id.xcc_id);
 446         mutex_unlock(&adev->grbm_idx_mutex);
 447
 448         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 449         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 450
 451         if (!x) {
 452                 result = -EINVAL;
 453                 goto done;
 454         }
 455
 456         while (size && (*pos < x * 4)) {
 457                 uint32_t value;
 458
 459                 value = data[*pos >> 2];
 460                 r = put_user(value, (uint32_t *)buf);
 461                 if (r) {
 462                         result = r;
 463                         goto done;
 464                 }
 465
 466                 result += 4;
 467                 buf += 4;
 468                 *pos += 4;
 469                 size -= 4;
 470         }
 471
 472 done:
 473         amdgpu_virt_disable_access_debugfs(adev);
 474         kfree(data);
 475         return result;
 476 }
 477
 478 static long amdgpu_debugfs_gprwave_ioctl(struct file *f, unsigned int cmd, unsigned long data)
 479 {
 480         struct amdgpu_debugfs_gprwave_data *rd = f->private_data;
 481         int r = 0;
 482
 483         mutex_lock(&rd->lock);
 484
 485         switch (cmd) {
 486         case AMDGPU_DEBUGFS_GPRWAVE_IOC_SET_STATE:
 487                 if (copy_from_user(&rd->id,
 488                                    (struct amdgpu_debugfs_gprwave_iocdata *)data,
 489                                    sizeof(rd->id)))
 490                         r = -EFAULT;
 491                 goto done;
 492         default:
 493                 r = -EINVAL;
 494                 goto done;
 495         }
 496
 497 done:
 498         mutex_unlock(&rd->lock);
 499         return r;
 500 }
 501
 502
 503
 504
 505 /**
 506  * amdgpu_debugfs_regs_pcie_read - Read from a PCIE register
 507  *
 508  * @f: open file handle
 509  * @buf: User buffer to store read data in
 510  * @size: Number of bytes to read
 511  * @pos:  Offset to seek to
 512  *
 513  * The lower bits are the BYTE offset of the register to read.  This
 514  * allows reading multiple registers in a single call and having
 515  * the returned size reflect that.
 516  */
 517 static ssize_t amdgpu_debugfs_regs_pcie_read(struct file *f, char __user *buf,
 518                                         size_t size, loff_t *pos)
 519 {
 520         struct amdgpu_device *adev = file_inode(f)->i_private;
 521         ssize_t result = 0;
 522         int r;
 523
 524         if (size & 0x3 || *pos & 0x3)
 525                 return -EINVAL;
 526
 527         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 528         if (r < 0) {
 529                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 530                 return r;
 531         }
 532
 533         r = amdgpu_virt_enable_access_debugfs(adev);
 534         if (r < 0) {
 535                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 536                 return r;
 537         }
 538
 539         while (size) {
 540                 uint32_t value;
 541
 542                 value = RREG32_PCIE(*pos);
 543                 r = put_user(value, (uint32_t *)buf);
 544                 if (r)
 545                         goto out;
 546
 547                 result += 4;
 548                 buf += 4;
 549                 *pos += 4;
 550                 size -= 4;
 551         }
 552
 553         r = result;
 554 out:
 555         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 556         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 557         amdgpu_virt_disable_access_debugfs(adev);
 558         return r;
 559 }
 560
 561 /**
 562  * amdgpu_debugfs_regs_pcie_write - Write to a PCIE register
 563  *
 564  * @f: open file handle
 565  * @buf: User buffer to write data from
 566  * @size: Number of bytes to write
 567  * @pos:  Offset to seek to
 568  *
 569  * The lower bits are the BYTE offset of the register to write.  This
 570  * allows writing multiple registers in a single call and having
 571  * the returned size reflect that.
 572  */
 573 static ssize_t amdgpu_debugfs_regs_pcie_write(struct file *f, const char __user *buf,
 574                                          size_t size, loff_t *pos)
 575 {
 576         struct amdgpu_device *adev = file_inode(f)->i_private;
 577         ssize_t result = 0;
 578         int r;
 579
 580         if (size & 0x3 || *pos & 0x3)
 581                 return -EINVAL;
 582
 583         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 584         if (r < 0) {
 585                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 586                 return r;
 587         }
 588
 589         r = amdgpu_virt_enable_access_debugfs(adev);
 590         if (r < 0) {
 591                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 592                 return r;
 593         }
 594
 595         while (size) {
 596                 uint32_t value;
 597
 598                 r = get_user(value, (uint32_t *)buf);
 599                 if (r)
 600                         goto out;
 601
 602                 WREG32_PCIE(*pos, value);
 603
 604                 result += 4;
 605                 buf += 4;
 606                 *pos += 4;
 607                 size -= 4;
 608         }
 609
 610         r = result;
 611 out:
 612         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 613         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 614         amdgpu_virt_disable_access_debugfs(adev);
 615         return r;
 616 }
 617
 618 /**
 619  * amdgpu_debugfs_regs_didt_read - Read from a DIDT register
 620  *
 621  * @f: open file handle
 622  * @buf: User buffer to store read data in
 623  * @size: Number of bytes to read
 624  * @pos:  Offset to seek to
 625  *
 626  * The lower bits are the BYTE offset of the register to read.  This
 627  * allows reading multiple registers in a single call and having
 628  * the returned size reflect that.
 629  */
 630 static ssize_t amdgpu_debugfs_regs_didt_read(struct file *f, char __user *buf,
 631                                         size_t size, loff_t *pos)
 632 {
 633         struct amdgpu_device *adev = file_inode(f)->i_private;
 634         ssize_t result = 0;
 635         int r;
 636
 637         if (size & 0x3 || *pos & 0x3)
 638                 return -EINVAL;
 639
 640         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 641         if (r < 0) {
 642                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 643                 return r;
 644         }
 645
 646         r = amdgpu_virt_enable_access_debugfs(adev);
 647         if (r < 0) {
 648                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 649                 return r;
 650         }
 651
 652         while (size) {
 653                 uint32_t value;
 654
 655                 value = RREG32_DIDT(*pos >> 2);
 656                 r = put_user(value, (uint32_t *)buf);
 657                 if (r)
 658                         goto out;
 659
 660                 result += 4;
 661                 buf += 4;
 662                 *pos += 4;
 663                 size -= 4;
 664         }
 665
 666         r = result;
 667 out:
 668         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 669         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 670         amdgpu_virt_disable_access_debugfs(adev);
 671         return r;
 672 }
 673
 674 /**
 675  * amdgpu_debugfs_regs_didt_write - Write to a DIDT register
 676  *
 677  * @f: open file handle
 678  * @buf: User buffer to write data from
 679  * @size: Number of bytes to write
 680  * @pos:  Offset to seek to
 681  *
 682  * The lower bits are the BYTE offset of the register to write.  This
 683  * allows writing multiple registers in a single call and having
 684  * the returned size reflect that.
 685  */
 686 static ssize_t amdgpu_debugfs_regs_didt_write(struct file *f, const char __user *buf,
 687                                          size_t size, loff_t *pos)
 688 {
 689         struct amdgpu_device *adev = file_inode(f)->i_private;
 690         ssize_t result = 0;
 691         int r;
 692
 693         if (size & 0x3 || *pos & 0x3)
 694                 return -EINVAL;
 695
 696         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 697         if (r < 0) {
 698                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 699                 return r;
 700         }
 701
 702         r = amdgpu_virt_enable_access_debugfs(adev);
 703         if (r < 0) {
 704                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 705                 return r;
 706         }
 707
 708         while (size) {
 709                 uint32_t value;
 710
 711                 r = get_user(value, (uint32_t *)buf);
 712                 if (r)
 713                         goto out;
 714
 715                 WREG32_DIDT(*pos >> 2, value);
 716
 717                 result += 4;
 718                 buf += 4;
 719                 *pos += 4;
 720                 size -= 4;
 721         }
 722
 723         r = result;
 724 out:
 725         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 726         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 727         amdgpu_virt_disable_access_debugfs(adev);
 728         return r;
 729 }
 730
 731 /**
 732  * amdgpu_debugfs_regs_smc_read - Read from a SMC register
 733  *
 734  * @f: open file handle
 735  * @buf: User buffer to store read data in
 736  * @size: Number of bytes to read
 737  * @pos:  Offset to seek to
 738  *
 739  * The lower bits are the BYTE offset of the register to read.  This
 740  * allows reading multiple registers in a single call and having
 741  * the returned size reflect that.
 742  */
 743 static ssize_t amdgpu_debugfs_regs_smc_read(struct file *f, char __user *buf,
 744                                         size_t size, loff_t *pos)
 745 {
 746         struct amdgpu_device *adev = file_inode(f)->i_private;
 747         ssize_t result = 0;
 748         int r;
 749
 750         if (size & 0x3 || *pos & 0x3)
 751                 return -EINVAL;
 752
 753         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 754         if (r < 0) {
 755                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 756                 return r;
 757         }
 758
 759         r = amdgpu_virt_enable_access_debugfs(adev);
 760         if (r < 0) {
 761                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 762                 return r;
 763         }
 764
 765         while (size) {
 766                 uint32_t value;
 767
 768                 value = RREG32_SMC(*pos);
 769                 r = put_user(value, (uint32_t *)buf);
 770                 if (r)
 771                         goto out;
 772
 773                 result += 4;
 774                 buf += 4;
 775                 *pos += 4;
 776                 size -= 4;
 777         }
 778
 779         r = result;
 780 out:
 781         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 782         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 783         amdgpu_virt_disable_access_debugfs(adev);
 784         return r;
 785 }
 786
 787 /**
 788  * amdgpu_debugfs_regs_smc_write - Write to a SMC register
 789  *
 790  * @f: open file handle
 791  * @buf: User buffer to write data from
 792  * @size: Number of bytes to write
 793  * @pos:  Offset to seek to
 794  *
 795  * The lower bits are the BYTE offset of the register to write.  This
 796  * allows writing multiple registers in a single call and having
 797  * the returned size reflect that.
 798  */
 799 static ssize_t amdgpu_debugfs_regs_smc_write(struct file *f, const char __user *buf,
 800                                          size_t size, loff_t *pos)
 801 {
 802         struct amdgpu_device *adev = file_inode(f)->i_private;
 803         ssize_t result = 0;
 804         int r;
 805
 806         if (size & 0x3 || *pos & 0x3)
 807                 return -EINVAL;
 808
 809         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 810         if (r < 0) {
 811                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 812                 return r;
 813         }
 814
 815         r = amdgpu_virt_enable_access_debugfs(adev);
 816         if (r < 0) {
 817                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 818                 return r;
 819         }
 820
 821         while (size) {
 822                 uint32_t value;
 823
 824                 r = get_user(value, (uint32_t *)buf);
 825                 if (r)
 826                         goto out;
 827
 828                 WREG32_SMC(*pos, value);
 829
 830                 result += 4;
 831                 buf += 4;
 832                 *pos += 4;
 833                 size -= 4;
 834         }
 835
 836         r = result;
 837 out:
 838         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 839         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 840         amdgpu_virt_disable_access_debugfs(adev);
 841         return r;
 842 }
 843
 844 /**
 845  * amdgpu_debugfs_gca_config_read - Read from gfx config data
 846  *
 847  * @f: open file handle
 848  * @buf: User buffer to store read data in
 849  * @size: Number of bytes to read
 850  * @pos:  Offset to seek to
 851  *
 852  * This file is used to access configuration data in a somewhat
 853  * stable fashion.  The format is a series of DWORDs with the first
 854  * indicating which revision it is.  New content is appended to the
 855  * end so that older software can still read the data.
 856  */
 857
 858 static ssize_t amdgpu_debugfs_gca_config_read(struct file *f, char __user *buf,
 859                                         size_t size, loff_t *pos)
 860 {
 861         struct amdgpu_device *adev = file_inode(f)->i_private;
 862         ssize_t result = 0;
 863         int r;
 864         uint32_t *config, no_regs = 0;
 865
 866         if (size & 0x3 || *pos & 0x3)
 867                 return -EINVAL;
 868
 869         config = kmalloc_array(256, sizeof(*config), GFP_KERNEL);
 870         if (!config)
 871                 return -ENOMEM;
 872
 873         /* version, increment each time something is added */
 874         config[no_regs++] = 5;
 875         config[no_regs++] = adev->gfx.config.max_shader_engines;
 876         config[no_regs++] = adev->gfx.config.max_tile_pipes;
 877         config[no_regs++] = adev->gfx.config.max_cu_per_sh;
 878         config[no_regs++] = adev->gfx.config.max_sh_per_se;
 879         config[no_regs++] = adev->gfx.config.max_backends_per_se;
 880         config[no_regs++] = adev->gfx.config.max_texture_channel_caches;
 881         config[no_regs++] = adev->gfx.config.max_gprs;
 882         config[no_regs++] = adev->gfx.config.max_gs_threads;
 883         config[no_regs++] = adev->gfx.config.max_hw_contexts;
 884         config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_frontend;
 885         config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_backend;
 886         config[no_regs++] = adev->gfx.config.sc_hiz_tile_fifo_size;
 887         config[no_regs++] = adev->gfx.config.sc_earlyz_tile_fifo_size;
 888         config[no_regs++] = adev->gfx.config.num_tile_pipes;
 889         config[no_regs++] = adev->gfx.config.backend_enable_mask;
 890         config[no_regs++] = adev->gfx.config.mem_max_burst_length_bytes;
 891         config[no_regs++] = adev->gfx.config.mem_row_size_in_kb;
 892         config[no_regs++] = adev->gfx.config.shader_engine_tile_size;
 893         config[no_regs++] = adev->gfx.config.num_gpus;
 894         config[no_regs++] = adev->gfx.config.multi_gpu_tile_size;
 895         config[no_regs++] = adev->gfx.config.mc_arb_ramcfg;
 896         config[no_regs++] = adev->gfx.config.gb_addr_config;
 897         config[no_regs++] = adev->gfx.config.num_rbs;
 898
 899         /* rev==1 */
 900         config[no_regs++] = adev->rev_id;
 901         config[no_regs++] = lower_32_bits(adev->pg_flags);
 902         config[no_regs++] = lower_32_bits(adev->cg_flags);
 903
 904         /* rev==2 */
 905         config[no_regs++] = adev->family;
 906         config[no_regs++] = adev->external_rev_id;
 907
 908         /* rev==3 */
 909         config[no_regs++] = adev->pdev->device;
 910         config[no_regs++] = adev->pdev->revision;
 911         config[no_regs++] = adev->pdev->subsystem_device;
 912         config[no_regs++] = adev->pdev->subsystem_vendor;
 913
 914         /* rev==4 APU flag */
 915         config[no_regs++] = adev->flags & AMD_IS_APU ? 1 : 0;
 916
 917         /* rev==5 PG/CG flag upper 32bit */
 918         config[no_regs++] = upper_32_bits(adev->pg_flags);
 919         config[no_regs++] = upper_32_bits(adev->cg_flags);
 920
 921         while (size && (*pos < no_regs * 4)) {
 922                 uint32_t value;
 923
 924                 value = config[*pos >> 2];
 925                 r = put_user(value, (uint32_t *)buf);
 926                 if (r) {
 927                         kfree(config);
 928                         return r;
 929                 }
 930
 931                 result += 4;
 932                 buf += 4;
 933                 *pos += 4;
 934                 size -= 4;
 935         }
 936
 937         kfree(config);
 938         return result;
 939 }
 940
 941 /**
 942  * amdgpu_debugfs_sensor_read - Read from the powerplay sensors
 943  *
 944  * @f: open file handle
 945  * @buf: User buffer to store read data in
 946  * @size: Number of bytes to read
 947  * @pos:  Offset to seek to
 948  *
 949  * The offset is treated as the BYTE address of one of the sensors
 950  * enumerated in amd/include/kgd_pp_interface.h under the
 951  * 'amd_pp_sensors' enumeration.  For instance to read the UVD VCLK
 952  * you would use the offset 3 * 4 = 12.
 953  */
 954 static ssize_t amdgpu_debugfs_sensor_read(struct file *f, char __user *buf,
 955                                         size_t size, loff_t *pos)
 956 {
 957         struct amdgpu_device *adev = file_inode(f)->i_private;
 958         int idx, x, outsize, r, valuesize;
 959         uint32_t values[16];
 960
 961         if (size & 3 || *pos & 0x3)
 962                 return -EINVAL;
 963
 964         if (!adev->pm.dpm_enabled)
 965                 return -EINVAL;
 966
 967         /* convert offset to sensor number */
 968         idx = *pos >> 2;
 969
 970         valuesize = sizeof(values);
 971
 972         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 973         if (r < 0) {
 974                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 975                 return r;
 976         }
 977
 978         r = amdgpu_virt_enable_access_debugfs(adev);
 979         if (r < 0) {
 980                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 981                 return r;
 982         }
 983
 984         r = amdgpu_dpm_read_sensor(adev, idx, &values[0], &valuesize);
 985
 986         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 987         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 988
 989         if (r) {
 990                 amdgpu_virt_disable_access_debugfs(adev);
 991                 return r;
 992         }
 993
 994         if (size > valuesize) {
 995                 amdgpu_virt_disable_access_debugfs(adev);
 996                 return -EINVAL;
 997         }
 998
 999         outsize = 0;
1000         x = 0;
1001         if (!r) {
1002                 while (size) {
1003                         r = put_user(values[x++], (int32_t *)buf);
1004                         buf += 4;
1005                         size -= 4;
1006                         outsize += 4;
1007                 }
1008         }
1009
1010         amdgpu_virt_disable_access_debugfs(adev);
1011         return !r ? outsize : r;
1012 }
1013
1014 /** amdgpu_debugfs_wave_read - Read WAVE STATUS data
1015  *
1016  * @f: open file handle
1017  * @buf: User buffer to store read data in
1018  * @size: Number of bytes to read
1019  * @pos:  Offset to seek to
1020  *
1021  * The offset being sought changes which wave that the status data
1022  * will be returned for.  The bits are used as follows:
1023  *
1024  * Bits 0..6:   Byte offset into data
1025  * Bits 7..14:  SE selector
1026  * Bits 15..22: SH/SA selector
1027  * Bits 23..30: CU/{WGP+SIMD} selector
1028  * Bits 31..36: WAVE ID selector
1029  * Bits 37..44: SIMD ID selector
1030  *
1031  * The returned data begins with one DWORD of version information
1032  * Followed by WAVE STATUS registers relevant to the GFX IP version
1033  * being used.  See gfx_v8_0_read_wave_data() for an example output.
1034  */
1035 static ssize_t amdgpu_debugfs_wave_read(struct file *f, char __user *buf,
1036                                         size_t size, loff_t *pos)
1037 {
1038         struct amdgpu_device *adev = f->f_inode->i_private;
1039         int r, x;
1040         ssize_t result = 0;
1041         uint32_t offset, se, sh, cu, wave, simd, data[32];
1042
1043         if (size & 3 || *pos & 3)
1044                 return -EINVAL;
1045
1046         /* decode offset */
1047         offset = (*pos & GENMASK_ULL(6, 0));
1048         se = (*pos & GENMASK_ULL(14, 7)) >> 7;
1049         sh = (*pos & GENMASK_ULL(22, 15)) >> 15;
1050         cu = (*pos & GENMASK_ULL(30, 23)) >> 23;
1051         wave = (*pos & GENMASK_ULL(36, 31)) >> 31;
1052         simd = (*pos & GENMASK_ULL(44, 37)) >> 37;
1053
1054         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1055         if (r < 0) {
1056                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1057                 return r;
1058         }
1059
1060         r = amdgpu_virt_enable_access_debugfs(adev);
1061         if (r < 0) {
1062                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1063                 return r;
1064         }
1065
1066         /* switch to the specific se/sh/cu */
1067         mutex_lock(&adev->grbm_idx_mutex);
1068         amdgpu_gfx_select_se_sh(adev, se, sh, cu, 0);
1069
1070         x = 0;
1071         if (adev->gfx.funcs->read_wave_data)
1072                 adev->gfx.funcs->read_wave_data(adev, 0, simd, wave, data, &x);
1073
1074         amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0);
1075         mutex_unlock(&adev->grbm_idx_mutex);
1076
1077         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1078         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1079
1080         if (!x) {
1081                 amdgpu_virt_disable_access_debugfs(adev);
1082                 return -EINVAL;
1083         }
1084
1085         while (size && (offset < x * 4)) {
1086                 uint32_t value;
1087
1088                 value = data[offset >> 2];
1089                 r = put_user(value, (uint32_t *)buf);
1090                 if (r) {
1091                         amdgpu_virt_disable_access_debugfs(adev);
1092                         return r;
1093                 }
1094
1095                 result += 4;
1096                 buf += 4;
1097                 offset += 4;
1098                 size -= 4;
1099         }
1100
1101         amdgpu_virt_disable_access_debugfs(adev);
1102         return result;
1103 }
1104
1105 /** amdgpu_debugfs_gpr_read - Read wave gprs
1106  *
1107  * @f: open file handle
1108  * @buf: User buffer to store read data in
1109  * @size: Number of bytes to read
1110  * @pos:  Offset to seek to
1111  *
1112  * The offset being sought changes which wave that the status data
1113  * will be returned for.  The bits are used as follows:
1114  *
1115  * Bits 0..11:  Byte offset into data
1116  * Bits 12..19: SE selector
1117  * Bits 20..27: SH/SA selector
1118  * Bits 28..35: CU/{WGP+SIMD} selector
1119  * Bits 36..43: WAVE ID selector
1120  * Bits 37..44: SIMD ID selector
1121  * Bits 52..59: Thread selector
1122  * Bits 60..61: Bank selector (VGPR=0,SGPR=1)
1123  *
1124  * The return data comes from the SGPR or VGPR register bank for
1125  * the selected operational unit.
1126  */
1127 static ssize_t amdgpu_debugfs_gpr_read(struct file *f, char __user *buf,
1128                                         size_t size, loff_t *pos)
1129 {
1130         struct amdgpu_device *adev = f->f_inode->i_private;
1131         int r;
1132         ssize_t result = 0;
1133         uint32_t offset, se, sh, cu, wave, simd, thread, bank, *data;
1134
1135         if (size > 4096 || size & 3 || *pos & 3)
1136                 return -EINVAL;
1137
1138         /* decode offset */
1139         offset = (*pos & GENMASK_ULL(11, 0)) >> 2;
1140         se = (*pos & GENMASK_ULL(19, 12)) >> 12;
1141         sh = (*pos & GENMASK_ULL(27, 20)) >> 20;
1142         cu = (*pos & GENMASK_ULL(35, 28)) >> 28;
1143         wave = (*pos & GENMASK_ULL(43, 36)) >> 36;
1144         simd = (*pos & GENMASK_ULL(51, 44)) >> 44;
1145         thread = (*pos & GENMASK_ULL(59, 52)) >> 52;
1146         bank = (*pos & GENMASK_ULL(61, 60)) >> 60;
1147
1148         data = kcalloc(1024, sizeof(*data), GFP_KERNEL);
1149         if (!data)
1150                 return -ENOMEM;
1151
1152         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1153         if (r < 0)
1154                 goto err;
1155
1156         r = amdgpu_virt_enable_access_debugfs(adev);
1157         if (r < 0)
1158                 goto err;
1159
1160         /* switch to the specific se/sh/cu */
1161         mutex_lock(&adev->grbm_idx_mutex);
1162         amdgpu_gfx_select_se_sh(adev, se, sh, cu, 0);
1163
1164         if (bank == 0) {
1165                 if (adev->gfx.funcs->read_wave_vgprs)
1166                         adev->gfx.funcs->read_wave_vgprs(adev, 0, simd, wave, thread, offset, size>>2, data);
1167         } else {
1168                 if (adev->gfx.funcs->read_wave_sgprs)
1169                         adev->gfx.funcs->read_wave_sgprs(adev, 0, simd, wave, offset, size>>2, data);
1170         }
1171
1172         amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0);
1173         mutex_unlock(&adev->grbm_idx_mutex);
1174
1175         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1176         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1177
1178         while (size) {
1179                 uint32_t value;
1180
1181                 value = data[result >> 2];
1182                 r = put_user(value, (uint32_t *)buf);
1183                 if (r) {
1184                         amdgpu_virt_disable_access_debugfs(adev);
1185                         goto err;
1186                 }
1187
1188                 result += 4;
1189                 buf += 4;
1190                 size -= 4;
1191         }
1192
1193         kfree(data);
1194         amdgpu_virt_disable_access_debugfs(adev);
1195         return result;
1196
1197 err:
1198         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1199         kfree(data);
1200         return r;
1201 }
1202
1203 /**
1204  * amdgpu_debugfs_gfxoff_residency_read - Read GFXOFF residency
1205  *
1206  * @f: open file handle
1207  * @buf: User buffer to store read data in
1208  * @size: Number of bytes to read
1209  * @pos:  Offset to seek to
1210  *
1211  * Read the last residency value logged. It doesn't auto update, one needs to
1212  * stop logging before getting the current value.
1213  */
1214 static ssize_t amdgpu_debugfs_gfxoff_residency_read(struct file *f, char __user *buf,
1215                                                     size_t size, loff_t *pos)
1216 {
1217         struct amdgpu_device *adev = file_inode(f)->i_private;
1218         ssize_t result = 0;
1219         int r;
1220
1221         if (size & 0x3 || *pos & 0x3)
1222                 return -EINVAL;
1223
1224         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1225         if (r < 0) {
1226                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1227                 return r;
1228         }
1229
1230         while (size) {
1231                 uint32_t value;
1232
1233                 r = amdgpu_get_gfx_off_residency(adev, &value);
1234                 if (r)
1235                         goto out;
1236
1237                 r = put_user(value, (uint32_t *)buf);
1238                 if (r)
1239                         goto out;
1240
1241                 result += 4;
1242                 buf += 4;
1243                 *pos += 4;
1244                 size -= 4;
1245         }
1246
1247         r = result;
1248 out:
1249         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1250         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1251
1252         return r;
1253 }
1254
1255 /**
1256  * amdgpu_debugfs_gfxoff_residency_write - Log GFXOFF Residency
1257  *
1258  * @f: open file handle
1259  * @buf: User buffer to write data from
1260  * @size: Number of bytes to write
1261  * @pos:  Offset to seek to
1262  *
1263  * Write a 32-bit non-zero to start logging; write a 32-bit zero to stop
1264  */
1265 static ssize_t amdgpu_debugfs_gfxoff_residency_write(struct file *f, const char __user *buf,
1266                                                      size_t size, loff_t *pos)
1267 {
1268         struct amdgpu_device *adev = file_inode(f)->i_private;
1269         ssize_t result = 0;
1270         int r;
1271
1272         if (size & 0x3 || *pos & 0x3)
1273                 return -EINVAL;
1274
1275         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1276         if (r < 0) {
1277                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1278                 return r;
1279         }
1280
1281         while (size) {
1282                 u32 value;
1283
1284                 r = get_user(value, (uint32_t *)buf);
1285                 if (r)
1286                         goto out;
1287
1288                 amdgpu_set_gfx_off_residency(adev, value ? true : false);
1289
1290                 result += 4;
1291                 buf += 4;
1292                 *pos += 4;
1293                 size -= 4;
1294         }
1295
1296         r = result;
1297 out:
1298         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1299         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1300
1301         return r;
1302 }
1303
1304
1305 /**
1306  * amdgpu_debugfs_gfxoff_count_read - Read GFXOFF entry count
1307  *
1308  * @f: open file handle
1309  * @buf: User buffer to store read data in
1310  * @size: Number of bytes to read
1311  * @pos:  Offset to seek to
1312  */
1313 static ssize_t amdgpu_debugfs_gfxoff_count_read(struct file *f, char __user *buf,
1314                                                 size_t size, loff_t *pos)
1315 {
1316         struct amdgpu_device *adev = file_inode(f)->i_private;
1317         ssize_t result = 0;
1318         int r;
1319
1320         if (size & 0x3 || *pos & 0x3)
1321                 return -EINVAL;
1322
1323         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1324         if (r < 0) {
1325                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1326                 return r;
1327         }
1328
1329         while (size) {
1330                 u64 value = 0;
1331
1332                 r = amdgpu_get_gfx_off_entrycount(adev, &value);
1333                 if (r)
1334                         goto out;
1335
1336                 r = put_user(value, (u64 *)buf);
1337                 if (r)
1338                         goto out;
1339
1340                 result += 4;
1341                 buf += 4;
1342                 *pos += 4;
1343                 size -= 4;
1344         }
1345
1346         r = result;
1347 out:
1348         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1349         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1350
1351         return r;
1352 }
1353
1354 /**
1355  * amdgpu_debugfs_gfxoff_write - Enable/disable GFXOFF
1356  *
1357  * @f: open file handle
1358  * @buf: User buffer to write data from
1359  * @size: Number of bytes to write
1360  * @pos:  Offset to seek to
1361  *
1362  * Write a 32-bit zero to disable or a 32-bit non-zero to enable
1363  */
1364 static ssize_t amdgpu_debugfs_gfxoff_write(struct file *f, const char __user *buf,
1365                                          size_t size, loff_t *pos)
1366 {
1367         struct amdgpu_device *adev = file_inode(f)->i_private;
1368         ssize_t result = 0;
1369         int r;
1370
1371         if (size & 0x3 || *pos & 0x3)
1372                 return -EINVAL;
1373
1374         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1375         if (r < 0) {
1376                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1377                 return r;
1378         }
1379
1380         while (size) {
1381                 uint32_t value;
1382
1383                 r = get_user(value, (uint32_t *)buf);
1384                 if (r)
1385                         goto out;
1386
1387                 amdgpu_gfx_off_ctrl(adev, value ? true : false);
1388
1389                 result += 4;
1390                 buf += 4;
1391                 *pos += 4;
1392                 size -= 4;
1393         }
1394
1395         r = result;
1396 out:
1397         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1398         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1399
1400         return r;
1401 }
1402
1403
1404 /**
1405  * amdgpu_debugfs_gfxoff_read - read gfxoff status
1406  *
1407  * @f: open file handle
1408  * @buf: User buffer to store read data in
1409  * @size: Number of bytes to read
1410  * @pos:  Offset to seek to
1411  */
1412 static ssize_t amdgpu_debugfs_gfxoff_read(struct file *f, char __user *buf,
1413                                          size_t size, loff_t *pos)
1414 {
1415         struct amdgpu_device *adev = file_inode(f)->i_private;
1416         ssize_t result = 0;
1417         int r;
1418
1419         if (size & 0x3 || *pos & 0x3)
1420                 return -EINVAL;
1421
1422         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1423         if (r < 0) {
1424                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1425                 return r;
1426         }
1427
1428         while (size) {
1429                 u32 value = adev->gfx.gfx_off_state;
1430
1431                 r = put_user(value, (u32 *)buf);
1432                 if (r)
1433                         goto out;
1434
1435                 result += 4;
1436                 buf += 4;
1437                 *pos += 4;
1438                 size -= 4;
1439         }
1440
1441         r = result;
1442 out:
1443         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1444         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1445
1446         return r;
1447 }
1448
1449 static ssize_t amdgpu_debugfs_gfxoff_status_read(struct file *f, char __user *buf,
1450                                                  size_t size, loff_t *pos)
1451 {
1452         struct amdgpu_device *adev = file_inode(f)->i_private;
1453         ssize_t result = 0;
1454         int r;
1455
1456         if (size & 0x3 || *pos & 0x3)
1457                 return -EINVAL;
1458
1459         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1460         if (r < 0) {
1461                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1462                 return r;
1463         }
1464
1465         while (size) {
1466                 u32 value;
1467
1468                 r = amdgpu_get_gfx_off_status(adev, &value);
1469                 if (r)
1470                         goto out;
1471
1472                 r = put_user(value, (u32 *)buf);
1473                 if (r)
1474                         goto out;
1475
1476                 result += 4;
1477                 buf += 4;
1478                 *pos += 4;
1479                 size -= 4;
1480         }
1481
1482         r = result;
1483 out:
1484         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1485         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1486
1487         return r;
1488 }
1489
1490 static const struct file_operations amdgpu_debugfs_regs2_fops = {
1491         .owner = THIS_MODULE,
1492         .unlocked_ioctl = amdgpu_debugfs_regs2_ioctl,
1493         .read = amdgpu_debugfs_regs2_read,
1494         .write = amdgpu_debugfs_regs2_write,
1495         .open = amdgpu_debugfs_regs2_open,
1496         .release = amdgpu_debugfs_regs2_release,
1497         .llseek = default_llseek
1498 };
1499
1500 static const struct file_operations amdgpu_debugfs_gprwave_fops = {
1501         .owner = THIS_MODULE,
1502         .unlocked_ioctl = amdgpu_debugfs_gprwave_ioctl,
1503         .read = amdgpu_debugfs_gprwave_read,
1504         .open = amdgpu_debugfs_gprwave_open,
1505         .release = amdgpu_debugfs_gprwave_release,
1506         .llseek = default_llseek
1507 };
1508
1509 static const struct file_operations amdgpu_debugfs_regs_fops = {
1510         .owner = THIS_MODULE,
1511         .read = amdgpu_debugfs_regs_read,
1512         .write = amdgpu_debugfs_regs_write,
1513         .llseek = default_llseek
1514 };
1515 static const struct file_operations amdgpu_debugfs_regs_didt_fops = {
1516         .owner = THIS_MODULE,
1517         .read = amdgpu_debugfs_regs_didt_read,
1518         .write = amdgpu_debugfs_regs_didt_write,
1519         .llseek = default_llseek
1520 };
1521 static const struct file_operations amdgpu_debugfs_regs_pcie_fops = {
1522         .owner = THIS_MODULE,
1523         .read = amdgpu_debugfs_regs_pcie_read,
1524         .write = amdgpu_debugfs_regs_pcie_write,
1525         .llseek = default_llseek
1526 };
1527 static const struct file_operations amdgpu_debugfs_regs_smc_fops = {
1528         .owner = THIS_MODULE,
1529         .read = amdgpu_debugfs_regs_smc_read,
1530         .write = amdgpu_debugfs_regs_smc_write,
1531         .llseek = default_llseek
1532 };
1533
1534 static const struct file_operations amdgpu_debugfs_gca_config_fops = {
1535         .owner = THIS_MODULE,
1536         .read = amdgpu_debugfs_gca_config_read,
1537         .llseek = default_llseek
1538 };
1539
1540 static const struct file_operations amdgpu_debugfs_sensors_fops = {
1541         .owner = THIS_MODULE,
1542         .read = amdgpu_debugfs_sensor_read,
1543         .llseek = default_llseek
1544 };
1545
1546 static const struct file_operations amdgpu_debugfs_wave_fops = {
1547         .owner = THIS_MODULE,
1548         .read = amdgpu_debugfs_wave_read,
1549         .llseek = default_llseek
1550 };
1551 static const struct file_operations amdgpu_debugfs_gpr_fops = {
1552         .owner = THIS_MODULE,
1553         .read = amdgpu_debugfs_gpr_read,
1554         .llseek = default_llseek
1555 };
1556
1557 static const struct file_operations amdgpu_debugfs_gfxoff_fops = {
1558         .owner = THIS_MODULE,
1559         .read = amdgpu_debugfs_gfxoff_read,
1560         .write = amdgpu_debugfs_gfxoff_write,
1561         .llseek = default_llseek
1562 };
1563
1564 static const struct file_operations amdgpu_debugfs_gfxoff_status_fops = {
1565         .owner = THIS_MODULE,
1566         .read = amdgpu_debugfs_gfxoff_status_read,
1567         .llseek = default_llseek
1568 };
1569
1570 static const struct file_operations amdgpu_debugfs_gfxoff_count_fops = {
1571         .owner = THIS_MODULE,
1572         .read = amdgpu_debugfs_gfxoff_count_read,
1573         .llseek = default_llseek
1574 };
1575
1576 static const struct file_operations amdgpu_debugfs_gfxoff_residency_fops = {
1577         .owner = THIS_MODULE,
1578         .read = amdgpu_debugfs_gfxoff_residency_read,
1579         .write = amdgpu_debugfs_gfxoff_residency_write,
1580         .llseek = default_llseek
1581 };
1582
1583 static const struct file_operations *debugfs_regs[] = {
1584         &amdgpu_debugfs_regs_fops,
1585         &amdgpu_debugfs_regs2_fops,
1586         &amdgpu_debugfs_gprwave_fops,
1587         &amdgpu_debugfs_regs_didt_fops,
1588         &amdgpu_debugfs_regs_pcie_fops,
1589         &amdgpu_debugfs_regs_smc_fops,
1590         &amdgpu_debugfs_gca_config_fops,
1591         &amdgpu_debugfs_sensors_fops,
1592         &amdgpu_debugfs_wave_fops,
1593         &amdgpu_debugfs_gpr_fops,
1594         &amdgpu_debugfs_gfxoff_fops,
1595         &amdgpu_debugfs_gfxoff_status_fops,
1596         &amdgpu_debugfs_gfxoff_count_fops,
1597         &amdgpu_debugfs_gfxoff_residency_fops,
1598 };
1599
1600 static const char * const debugfs_regs_names[] = {
1601         "amdgpu_regs",
1602         "amdgpu_regs2",
1603         "amdgpu_gprwave",
1604         "amdgpu_regs_didt",
1605         "amdgpu_regs_pcie",
1606         "amdgpu_regs_smc",
1607         "amdgpu_gca_config",
1608         "amdgpu_sensors",
1609         "amdgpu_wave",
1610         "amdgpu_gpr",
1611         "amdgpu_gfxoff",
1612         "amdgpu_gfxoff_status",
1613         "amdgpu_gfxoff_count",
1614         "amdgpu_gfxoff_residency",
1615 };
1616
1617 /**
1618  * amdgpu_debugfs_regs_init -   Initialize debugfs entries that provide
1619  *                              register access.
1620  *
1621  * @adev: The device to attach the debugfs entries to
1622  */
1623 int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
1624 {
1625         struct drm_minor *minor = adev_to_drm(adev)->primary;
1626         struct dentry *ent, *root = minor->debugfs_root;
1627         unsigned int i;
1628
1629         for (i = 0; i < ARRAY_SIZE(debugfs_regs); i++) {
1630                 ent = debugfs_create_file(debugfs_regs_names[i],
1631                                           S_IFREG | 0444, root,
1632                                           adev, debugfs_regs[i]);
1633                 if (!i && !IS_ERR_OR_NULL(ent))
1634                         i_size_write(ent->d_inode, adev->rmmio_size);
1635         }
1636
1637         return 0;
1638 }
1639
1640 static int amdgpu_debugfs_test_ib_show(struct seq_file *m, void *unused)
1641 {
1642         struct amdgpu_device *adev = m->private;
1643         struct drm_device *dev = adev_to_drm(adev);
1644         int r = 0, i;
1645
1646         r = pm_runtime_get_sync(dev->dev);
1647         if (r < 0) {
1648                 pm_runtime_put_autosuspend(dev->dev);
1649                 return r;
1650         }
1651
1652         /* Avoid accidently unparking the sched thread during GPU reset */
1653         r = down_write_killable(&adev->reset_domain->sem);
1654         if (r)
1655                 return r;
1656
1657         /* hold on the scheduler */
1658         for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
1659                 struct amdgpu_ring *ring = adev->rings[i];
1660
1661                 if (!ring || !ring->sched.thread)
1662                         continue;
1663                 kthread_park(ring->sched.thread);
1664         }
1665
1666         seq_puts(m, "run ib test:\n");
1667         r = amdgpu_ib_ring_tests(adev);
1668         if (r)
1669                 seq_printf(m, "ib ring tests failed (%d).\n", r);
1670         else
1671                 seq_puts(m, "ib ring tests passed.\n");
1672
1673         /* go on the scheduler */
1674         for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
1675                 struct amdgpu_ring *ring = adev->rings[i];
1676
1677                 if (!ring || !ring->sched.thread)
1678                         continue;
1679                 kthread_unpark(ring->sched.thread);
1680         }
1681
1682         up_write(&adev->reset_domain->sem);
1683
1684         pm_runtime_mark_last_busy(dev->dev);
1685         pm_runtime_put_autosuspend(dev->dev);
1686
1687         return 0;
1688 }
1689
1690 static int amdgpu_debugfs_evict_vram(void *data, u64 *val)
1691 {
1692         struct amdgpu_device *adev = (struct amdgpu_device *)data;
1693         struct drm_device *dev = adev_to_drm(adev);
1694         int r;
1695
1696         r = pm_runtime_get_sync(dev->dev);
1697         if (r < 0) {
1698                 pm_runtime_put_autosuspend(dev->dev);
1699                 return r;
1700         }
1701
1702         *val = amdgpu_ttm_evict_resources(adev, TTM_PL_VRAM);
1703
1704         pm_runtime_mark_last_busy(dev->dev);
1705         pm_runtime_put_autosuspend(dev->dev);
1706
1707         return 0;
1708 }
1709
1710
1711 static int amdgpu_debugfs_evict_gtt(void *data, u64 *val)
1712 {
1713         struct amdgpu_device *adev = (struct amdgpu_device *)data;
1714         struct drm_device *dev = adev_to_drm(adev);
1715         int r;
1716
1717         r = pm_runtime_get_sync(dev->dev);
1718         if (r < 0) {
1719                 pm_runtime_put_autosuspend(dev->dev);
1720                 return r;
1721         }
1722
1723         *val = amdgpu_ttm_evict_resources(adev, TTM_PL_TT);
1724
1725         pm_runtime_mark_last_busy(dev->dev);
1726         pm_runtime_put_autosuspend(dev->dev);
1727
1728         return 0;
1729 }
1730
1731 static int amdgpu_debugfs_benchmark(void *data, u64 val)
1732 {
1733         struct amdgpu_device *adev = (struct amdgpu_device *)data;
1734         struct drm_device *dev = adev_to_drm(adev);
1735         int r;
1736
1737         r = pm_runtime_get_sync(dev->dev);
1738         if (r < 0) {
1739                 pm_runtime_put_autosuspend(dev->dev);
1740                 return r;
1741         }
1742
1743         r = amdgpu_benchmark(adev, val);
1744
1745         pm_runtime_mark_last_busy(dev->dev);
1746         pm_runtime_put_autosuspend(dev->dev);
1747
1748         return r;
1749 }
1750
1751 static int amdgpu_debugfs_vm_info_show(struct seq_file *m, void *unused)
1752 {
1753         struct amdgpu_device *adev = m->private;
1754         struct drm_device *dev = adev_to_drm(adev);
1755         struct drm_file *file;
1756         int r;
1757
1758         r = mutex_lock_interruptible(&dev->filelist_mutex);
1759         if (r)
1760                 return r;
1761
1762         list_for_each_entry(file, &dev->filelist, lhead) {
1763                 struct amdgpu_fpriv *fpriv = file->driver_priv;
1764                 struct amdgpu_vm *vm = &fpriv->vm;
1765
1766                 seq_printf(m, "pid:%d\tProcess:%s ----------\n",
1767                                 vm->task_info.pid, vm->task_info.process_name);
1768                 r = amdgpu_bo_reserve(vm->root.bo, true);
1769                 if (r)
1770                         break;
1771                 amdgpu_debugfs_vm_bo_info(vm, m);
1772                 amdgpu_bo_unreserve(vm->root.bo);
1773         }
1774
1775         mutex_unlock(&dev->filelist_mutex);
1776
1777         return r;
1778 }
1779
1780 DEFINE_SHOW_ATTRIBUTE(amdgpu_debugfs_test_ib);
1781 DEFINE_SHOW_ATTRIBUTE(amdgpu_debugfs_vm_info);
1782 DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_evict_vram_fops, amdgpu_debugfs_evict_vram,
1783                          NULL, "%lld\n");
1784 DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_evict_gtt_fops, amdgpu_debugfs_evict_gtt,
1785                          NULL, "%lld\n");
1786 DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_benchmark_fops, NULL, amdgpu_debugfs_benchmark,
1787                          "%lld\n");
1788
1789 static void amdgpu_ib_preempt_fences_swap(struct amdgpu_ring *ring,
1790                                           struct dma_fence **fences)
1791 {
1792         struct amdgpu_fence_driver *drv = &ring->fence_drv;
1793         uint32_t sync_seq, last_seq;
1794
1795         last_seq = atomic_read(&ring->fence_drv.last_seq);
1796         sync_seq = ring->fence_drv.sync_seq;
1797
1798         last_seq &= drv->num_fences_mask;
1799         sync_seq &= drv->num_fences_mask;
1800
1801         do {
1802                 struct dma_fence *fence, **ptr;
1803
1804                 ++last_seq;
1805                 last_seq &= drv->num_fences_mask;
1806                 ptr = &drv->fences[last_seq];
1807
1808                 fence = rcu_dereference_protected(*ptr, 1);
1809                 RCU_INIT_POINTER(*ptr, NULL);
1810
1811                 if (!fence)
1812                         continue;
1813
1814                 fences[last_seq] = fence;
1815
1816         } while (last_seq != sync_seq);
1817 }
1818
1819 static void amdgpu_ib_preempt_signal_fences(struct dma_fence **fences,
1820                                             int length)
1821 {
1822         int i;
1823         struct dma_fence *fence;
1824
1825         for (i = 0; i < length; i++) {
1826                 fence = fences[i];
1827                 if (!fence)
1828                         continue;
1829                 dma_fence_signal(fence);
1830                 dma_fence_put(fence);
1831         }
1832 }
1833
1834 static void amdgpu_ib_preempt_job_recovery(struct drm_gpu_scheduler *sched)
1835 {
1836         struct drm_sched_job *s_job;
1837         struct dma_fence *fence;
1838
1839         spin_lock(&sched->job_list_lock);
1840         list_for_each_entry(s_job, &sched->pending_list, list) {
1841                 fence = sched->ops->run_job(s_job);
1842                 dma_fence_put(fence);
1843         }
1844         spin_unlock(&sched->job_list_lock);
1845 }
1846
1847 static void amdgpu_ib_preempt_mark_partial_job(struct amdgpu_ring *ring)
1848 {
1849         struct amdgpu_job *job;
1850         struct drm_sched_job *s_job, *tmp;
1851         uint32_t preempt_seq;
1852         struct dma_fence *fence, **ptr;
1853         struct amdgpu_fence_driver *drv = &ring->fence_drv;
1854         struct drm_gpu_scheduler *sched = &ring->sched;
1855         bool preempted = true;
1856
1857         if (ring->funcs->type != AMDGPU_RING_TYPE_GFX)
1858                 return;
1859
1860         preempt_seq = le32_to_cpu(*(drv->cpu_addr + 2));
1861         if (preempt_seq <= atomic_read(&drv->last_seq)) {
1862                 preempted = false;
1863                 goto no_preempt;
1864         }
1865
1866         preempt_seq &= drv->num_fences_mask;
1867         ptr = &drv->fences[preempt_seq];
1868         fence = rcu_dereference_protected(*ptr, 1);
1869
1870 no_preempt:
1871         spin_lock(&sched->job_list_lock);
1872         list_for_each_entry_safe(s_job, tmp, &sched->pending_list, list) {
1873                 if (dma_fence_is_signaled(&s_job->s_fence->finished)) {
1874                         /* remove job from ring_mirror_list */
1875                         list_del_init(&s_job->list);
1876                         sched->ops->free_job(s_job);
1877                         continue;
1878                 }
1879                 job = to_amdgpu_job(s_job);
1880                 if (preempted && (&job->hw_fence) == fence)
1881                         /* mark the job as preempted */
1882                         job->preemption_status |= AMDGPU_IB_PREEMPTED;
1883         }
1884         spin_unlock(&sched->job_list_lock);
1885 }
1886
1887 static int amdgpu_debugfs_ib_preempt(void *data, u64 val)
1888 {
1889         int r, length;
1890         struct amdgpu_ring *ring;
1891         struct dma_fence **fences = NULL;
1892         struct amdgpu_device *adev = (struct amdgpu_device *)data;
1893
1894         if (val >= AMDGPU_MAX_RINGS)
1895                 return -EINVAL;
1896
1897         ring = adev->rings[val];
1898
1899         if (!ring || !ring->funcs->preempt_ib || !ring->sched.thread)
1900                 return -EINVAL;
1901
1902         /* the last preemption failed */
1903         if (ring->trail_seq != le32_to_cpu(*ring->trail_fence_cpu_addr))
1904                 return -EBUSY;
1905
1906         length = ring->fence_drv.num_fences_mask + 1;
1907         fences = kcalloc(length, sizeof(void *), GFP_KERNEL);
1908         if (!fences)
1909                 return -ENOMEM;
1910
1911         /* Avoid accidently unparking the sched thread during GPU reset */
1912         r = down_read_killable(&adev->reset_domain->sem);
1913         if (r)
1914                 goto pro_end;
1915
1916         /* stop the scheduler */
1917         kthread_park(ring->sched.thread);
1918
1919         /* preempt the IB */
1920         r = amdgpu_ring_preempt_ib(ring);
1921         if (r) {
1922                 DRM_WARN("failed to preempt ring %d\n", ring->idx);
1923                 goto failure;
1924         }
1925
1926         amdgpu_fence_process(ring);
1927
1928         if (atomic_read(&ring->fence_drv.last_seq) !=
1929             ring->fence_drv.sync_seq) {
1930                 DRM_INFO("ring %d was preempted\n", ring->idx);
1931
1932                 amdgpu_ib_preempt_mark_partial_job(ring);
1933
1934                 /* swap out the old fences */
1935                 amdgpu_ib_preempt_fences_swap(ring, fences);
1936
1937                 amdgpu_fence_driver_force_completion(ring);
1938
1939                 /* resubmit unfinished jobs */
1940                 amdgpu_ib_preempt_job_recovery(&ring->sched);
1941
1942                 /* wait for jobs finished */
1943                 amdgpu_fence_wait_empty(ring);
1944
1945                 /* signal the old fences */
1946                 amdgpu_ib_preempt_signal_fences(fences, length);
1947         }
1948
1949 failure:
1950         /* restart the scheduler */
1951         kthread_unpark(ring->sched.thread);
1952
1953         up_read(&adev->reset_domain->sem);
1954
1955 pro_end:
1956         kfree(fences);
1957
1958         return r;
1959 }
1960
1961 static int amdgpu_debugfs_sclk_set(void *data, u64 val)
1962 {
1963         int ret = 0;
1964         uint32_t max_freq, min_freq;
1965         struct amdgpu_device *adev = (struct amdgpu_device *)data;
1966
1967         if (amdgpu_sriov_vf(adev) && !amdgpu_sriov_is_pp_one_vf(adev))
1968                 return -EINVAL;
1969
1970         ret = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1971         if (ret < 0) {
1972                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1973                 return ret;
1974         }
1975
1976         ret = amdgpu_dpm_get_dpm_freq_range(adev, PP_SCLK, &min_freq, &max_freq);
1977         if (ret == -EOPNOTSUPP) {
1978                 ret = 0;
1979                 goto out;
1980         }
1981         if (ret || val > max_freq || val < min_freq) {
1982                 ret = -EINVAL;
1983                 goto out;
1984         }
1985
1986         ret = amdgpu_dpm_set_soft_freq_range(adev, PP_SCLK, (uint32_t)val, (uint32_t)val);
1987         if (ret)
1988                 ret = -EINVAL;
1989
1990 out:
1991         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1992         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1993
1994         return ret;
1995 }
1996
1997 DEFINE_DEBUGFS_ATTRIBUTE(fops_ib_preempt, NULL,
1998                         amdgpu_debugfs_ib_preempt, "%llu\n");
1999
2000 DEFINE_DEBUGFS_ATTRIBUTE(fops_sclk_set, NULL,
2001                         amdgpu_debugfs_sclk_set, "%llu\n");
2002
2003 static ssize_t amdgpu_reset_dump_register_list_read(struct file *f,
2004                                 char __user *buf, size_t size, loff_t *pos)
2005 {
2006         struct amdgpu_device *adev = (struct amdgpu_device *)file_inode(f)->i_private;
2007         char reg_offset[12];
2008         int i, ret, len = 0;
2009
2010         if (*pos)
2011                 return 0;
2012
2013         memset(reg_offset, 0, 12);
2014         ret = down_read_killable(&adev->reset_domain->sem);
2015         if (ret)
2016                 return ret;
2017
2018         for (i = 0; i < adev->num_regs; i++) {
2019                 sprintf(reg_offset, "0x%x\n", adev->reset_dump_reg_list[i]);
2020                 up_read(&adev->reset_domain->sem);
2021                 if (copy_to_user(buf + len, reg_offset, strlen(reg_offset)))
2022                         return -EFAULT;
2023
2024                 len += strlen(reg_offset);
2025                 ret = down_read_killable(&adev->reset_domain->sem);
2026                 if (ret)
2027                         return ret;
2028         }
2029
2030         up_read(&adev->reset_domain->sem);
2031         *pos += len;
2032
2033         return len;
2034 }
2035
2036 static ssize_t amdgpu_reset_dump_register_list_write(struct file *f,
2037                         const char __user *buf, size_t size, loff_t *pos)
2038 {
2039         struct amdgpu_device *adev = (struct amdgpu_device *)file_inode(f)->i_private;
2040         char reg_offset[11];
2041         uint32_t *new = NULL, *tmp = NULL;
2042         int ret, i = 0, len = 0;
2043
2044         do {
2045                 memset(reg_offset, 0, 11);
2046                 if (copy_from_user(reg_offset, buf + len,
2047                                         min(10, ((int)size-len)))) {
2048                         ret = -EFAULT;
2049                         goto error_free;
2050                 }
2051
2052                 new = krealloc_array(tmp, i + 1, sizeof(uint32_t), GFP_KERNEL);
2053                 if (!new) {
2054                         ret = -ENOMEM;
2055                         goto error_free;
2056                 }
2057                 tmp = new;
2058                 if (sscanf(reg_offset, "%X %n", &tmp[i], &ret) != 1) {
2059                         ret = -EINVAL;
2060                         goto error_free;
2061                 }
2062
2063                 len += ret;
2064                 i++;
2065         } while (len < size);
2066
2067         new = kmalloc_array(i, sizeof(uint32_t), GFP_KERNEL);
2068         if (!new) {
2069                 ret = -ENOMEM;
2070                 goto error_free;
2071         }
2072         ret = down_write_killable(&adev->reset_domain->sem);
2073         if (ret)
2074                 goto error_free;
2075
2076         swap(adev->reset_dump_reg_list, tmp);
2077         swap(adev->reset_dump_reg_value, new);
2078         adev->num_regs = i;
2079         up_write(&adev->reset_domain->sem);
2080         ret = size;
2081
2082 error_free:
2083         if (tmp != new)
2084                 kfree(tmp);
2085         kfree(new);
2086         return ret;
2087 }
2088
2089 static const struct file_operations amdgpu_reset_dump_register_list = {
2090         .owner = THIS_MODULE,
2091         .read = amdgpu_reset_dump_register_list_read,
2092         .write = amdgpu_reset_dump_register_list_write,
2093         .llseek = default_llseek
2094 };
2095
2096 int amdgpu_debugfs_init(struct amdgpu_device *adev)
2097 {
2098         struct dentry *root = adev_to_drm(adev)->primary->debugfs_root;
2099         struct dentry *ent;
2100         int r, i;
2101
2102         if (!debugfs_initialized())
2103                 return 0;
2104
2105         debugfs_create_x32("amdgpu_smu_debug", 0600, root,
2106                            &adev->pm.smu_debug_mask);
2107
2108         ent = debugfs_create_file("amdgpu_preempt_ib", 0600, root, adev,
2109                                   &fops_ib_preempt);
2110         if (IS_ERR(ent)) {
2111                 DRM_ERROR("unable to create amdgpu_preempt_ib debugsfs file\n");
2112                 return PTR_ERR(ent);
2113         }
2114
2115         ent = debugfs_create_file("amdgpu_force_sclk", 0200, root, adev,
2116                                   &fops_sclk_set);
2117         if (IS_ERR(ent)) {
2118                 DRM_ERROR("unable to create amdgpu_set_sclk debugsfs file\n");
2119                 return PTR_ERR(ent);
2120         }
2121
2122         /* Register debugfs entries for amdgpu_ttm */
2123         amdgpu_ttm_debugfs_init(adev);
2124         amdgpu_debugfs_pm_init(adev);
2125         amdgpu_debugfs_sa_init(adev);
2126         amdgpu_debugfs_fence_init(adev);
2127         amdgpu_debugfs_gem_init(adev);
2128
2129         r = amdgpu_debugfs_regs_init(adev);
2130         if (r)
2131                 DRM_ERROR("registering register debugfs failed (%d).\n", r);
2132
2133         amdgpu_debugfs_firmware_init(adev);
2134         amdgpu_ta_if_debugfs_init(adev);
2135
2136 #if defined(CONFIG_DRM_AMD_DC)
2137         if (adev->dc_enabled)
2138                 dtn_debugfs_init(adev);
2139 #endif
2140
2141         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
2142                 struct amdgpu_ring *ring = adev->rings[i];
2143
2144                 if (!ring)
2145                         continue;
2146
2147                 amdgpu_debugfs_ring_init(adev, ring);
2148         }
2149
2150         for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
2151                 if (!amdgpu_vcnfw_log)
2152                         break;
2153
2154                 if (adev->vcn.harvest_config & (1 << i))
2155                         continue;
2156
2157                 amdgpu_debugfs_vcn_fwlog_init(adev, i, &adev->vcn.inst[i]);
2158         }
2159
2160         amdgpu_ras_debugfs_create_all(adev);
2161         amdgpu_rap_debugfs_init(adev);
2162         amdgpu_securedisplay_debugfs_init(adev);
2163         amdgpu_fw_attestation_debugfs_init(adev);
2164
2165         debugfs_create_file("amdgpu_evict_vram", 0444, root, adev,
2166                             &amdgpu_evict_vram_fops);
2167         debugfs_create_file("amdgpu_evict_gtt", 0444, root, adev,
2168                             &amdgpu_evict_gtt_fops);
2169         debugfs_create_file("amdgpu_test_ib", 0444, root, adev,
2170                             &amdgpu_debugfs_test_ib_fops);
2171         debugfs_create_file("amdgpu_vm_info", 0444, root, adev,
2172                             &amdgpu_debugfs_vm_info_fops);
2173         debugfs_create_file("amdgpu_benchmark", 0200, root, adev,
2174                             &amdgpu_benchmark_fops);
2175         debugfs_create_file("amdgpu_reset_dump_register_list", 0644, root, adev,
2176                             &amdgpu_reset_dump_register_list);
2177
2178         adev->debugfs_vbios_blob.data = adev->bios;
2179         adev->debugfs_vbios_blob.size = adev->bios_size;
2180         debugfs_create_blob("amdgpu_vbios", 0444, root,
2181                             &adev->debugfs_vbios_blob);
2182
2183         adev->debugfs_discovery_blob.data = adev->mman.discovery_bin;
2184         adev->debugfs_discovery_blob.size = adev->mman.discovery_tmr_size;
2185         debugfs_create_blob("amdgpu_discovery", 0444, root,
2186                             &adev->debugfs_discovery_blob);
2187
2188         return 0;
2189 }
2190
2191 #else
2192 int amdgpu_debugfs_init(struct amdgpu_device *adev)
2193 {
2194         return 0;
2195 }
2196 int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
2197 {
2198         return 0;
2199 }
2200 #endif