drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c

   1 /*
   2  * Copyright 2008 Advanced Micro Devices, Inc.
   3  * Copyright 2008 Red Hat Inc.
   4  * Copyright 2009 Jerome Glisse.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22  * OTHER DEALINGS IN THE SOFTWARE.
  23  *
  24  */
  25
  26 #include <linux/kthread.h>
  27 #include <linux/pci.h>
  28 #include <linux/uaccess.h>
  29 #include <linux/pm_runtime.h>
  30
  31 #include "amdgpu.h"
  32 #include "amdgpu_pm.h"
  33 #include "amdgpu_dm_debugfs.h"
  34 #include "amdgpu_ras.h"
  35 #include "amdgpu_rap.h"
  36 #include "amdgpu_securedisplay.h"
  37 #include "amdgpu_fw_attestation.h"
  38 #include "amdgpu_umr.h"
  39
  40 #include "amdgpu_reset.h"
  41 #include "amdgpu_psp_ta.h"
  42
  43 #if defined(CONFIG_DEBUG_FS)
  44
  45 /**
  46  * amdgpu_debugfs_process_reg_op - Handle MMIO register reads/writes
  47  *
  48  * @read: True if reading
  49  * @f: open file handle
  50  * @buf: User buffer to write/read to
  51  * @size: Number of bytes to write/read
  52  * @pos:  Offset to seek to
  53  *
  54  * This debugfs entry has special meaning on the offset being sought.
  55  * Various bits have different meanings:
  56  *
  57  * Bit 62:  Indicates a GRBM bank switch is needed
  58  * Bit 61:  Indicates a SRBM bank switch is needed (implies bit 62 is
  59  *          zero)
  60  * Bits 24..33: The SE or ME selector if needed
  61  * Bits 34..43: The SH (or SA) or PIPE selector if needed
  62  * Bits 44..53: The INSTANCE (or CU/WGP) or QUEUE selector if needed
  63  *
  64  * Bit 23:  Indicates that the PM power gating lock should be held
  65  *          This is necessary to read registers that might be
  66  *          unreliable during a power gating transistion.
  67  *
  68  * The lower bits are the BYTE offset of the register to read.  This
  69  * allows reading multiple registers in a single call and having
  70  * the returned size reflect that.
  71  */
  72 static int  amdgpu_debugfs_process_reg_op(bool read, struct file *f,
  73                 char __user *buf, size_t size, loff_t *pos)
  74 {
  75         struct amdgpu_device *adev = file_inode(f)->i_private;
  76         ssize_t result = 0;
  77         int r;
  78         bool pm_pg_lock, use_bank, use_ring;
  79         unsigned int instance_bank, sh_bank, se_bank, me, pipe, queue, vmid;
  80
  81         pm_pg_lock = use_bank = use_ring = false;
  82         instance_bank = sh_bank = se_bank = me = pipe = queue = vmid = 0;
  83
  84         if (size & 0x3 || *pos & 0x3 ||
  85                         ((*pos & (1ULL << 62)) && (*pos & (1ULL << 61))))
  86                 return -EINVAL;
  87
  88         /* are we reading registers for which a PG lock is necessary? */
  89         pm_pg_lock = (*pos >> 23) & 1;
  90
  91         if (*pos & (1ULL << 62)) {
  92                 se_bank = (*pos & GENMASK_ULL(33, 24)) >> 24;
  93                 sh_bank = (*pos & GENMASK_ULL(43, 34)) >> 34;
  94                 instance_bank = (*pos & GENMASK_ULL(53, 44)) >> 44;
  95
  96                 if (se_bank == 0x3FF)
  97                         se_bank = 0xFFFFFFFF;
  98                 if (sh_bank == 0x3FF)
  99                         sh_bank = 0xFFFFFFFF;
 100                 if (instance_bank == 0x3FF)
 101                         instance_bank = 0xFFFFFFFF;
 102                 use_bank = true;
 103         } else if (*pos & (1ULL << 61)) {
 104
 105                 me = (*pos & GENMASK_ULL(33, 24)) >> 24;
 106                 pipe = (*pos & GENMASK_ULL(43, 34)) >> 34;
 107                 queue = (*pos & GENMASK_ULL(53, 44)) >> 44;
 108                 vmid = (*pos & GENMASK_ULL(58, 54)) >> 54;
 109
 110                 use_ring = true;
 111         } else {
 112                 use_bank = use_ring = false;
 113         }
 114
 115         *pos &= (1UL << 22) - 1;
 116
 117         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 118         if (r < 0) {
 119                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 120                 return r;
 121         }
 122
 123         r = amdgpu_virt_enable_access_debugfs(adev);
 124         if (r < 0) {
 125                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 126                 return r;
 127         }
 128
 129         if (use_bank) {
 130                 if ((sh_bank != 0xFFFFFFFF && sh_bank >= adev->gfx.config.max_sh_per_se) ||
 131                     (se_bank != 0xFFFFFFFF && se_bank >= adev->gfx.config.max_shader_engines)) {
 132                         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 133                         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 134                         amdgpu_virt_disable_access_debugfs(adev);
 135                         return -EINVAL;
 136                 }
 137                 mutex_lock(&adev->grbm_idx_mutex);
 138                 amdgpu_gfx_select_se_sh(adev, se_bank,
 139                                         sh_bank, instance_bank, 0);
 140         } else if (use_ring) {
 141                 mutex_lock(&adev->srbm_mutex);
 142                 amdgpu_gfx_select_me_pipe_q(adev, me, pipe, queue, vmid, 0);
 143         }
 144
 145         if (pm_pg_lock)
 146                 mutex_lock(&adev->pm.mutex);
 147
 148         while (size) {
 149                 uint32_t value;
 150
 151                 if (read) {
 152                         value = RREG32(*pos >> 2);
 153                         r = put_user(value, (uint32_t *)buf);
 154                 } else {
 155                         r = get_user(value, (uint32_t *)buf);
 156                         if (!r)
 157                                 amdgpu_mm_wreg_mmio_rlc(adev, *pos >> 2, value);
 158                 }
 159                 if (r) {
 160                         result = r;
 161                         goto end;
 162                 }
 163
 164                 result += 4;
 165                 buf += 4;
 166                 *pos += 4;
 167                 size -= 4;
 168         }
 169
 170 end:
 171         if (use_bank) {
 172                 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
 173                 mutex_unlock(&adev->grbm_idx_mutex);
 174         } else if (use_ring) {
 175                 amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0);
 176                 mutex_unlock(&adev->srbm_mutex);
 177         }
 178
 179         if (pm_pg_lock)
 180                 mutex_unlock(&adev->pm.mutex);
 181
 182         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 183         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 184
 185         amdgpu_virt_disable_access_debugfs(adev);
 186         return result;
 187 }
 188
 189 /*
 190  * amdgpu_debugfs_regs_read - Callback for reading MMIO registers
 191  */
 192 static ssize_t amdgpu_debugfs_regs_read(struct file *f, char __user *buf,
 193                                         size_t size, loff_t *pos)
 194 {
 195         return amdgpu_debugfs_process_reg_op(true, f, buf, size, pos);
 196 }
 197
 198 /*
 199  * amdgpu_debugfs_regs_write - Callback for writing MMIO registers
 200  */
 201 static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,
 202                                          size_t size, loff_t *pos)
 203 {
 204         return amdgpu_debugfs_process_reg_op(false, f, (char __user *)buf, size, pos);
 205 }
 206
 207 static int amdgpu_debugfs_regs2_open(struct inode *inode, struct file *file)
 208 {
 209         struct amdgpu_debugfs_regs2_data *rd;
 210
 211         rd = kzalloc(sizeof(*rd), GFP_KERNEL);
 212         if (!rd)
 213                 return -ENOMEM;
 214         rd->adev = file_inode(file)->i_private;
 215         file->private_data = rd;
 216         mutex_init(&rd->lock);
 217
 218         return 0;
 219 }
 220
 221 static int amdgpu_debugfs_regs2_release(struct inode *inode, struct file *file)
 222 {
 223         struct amdgpu_debugfs_regs2_data *rd = file->private_data;
 224
 225         mutex_destroy(&rd->lock);
 226         kfree(file->private_data);
 227         return 0;
 228 }
 229
 230 static ssize_t amdgpu_debugfs_regs2_op(struct file *f, char __user *buf, u32 offset, size_t size, int write_en)
 231 {
 232         struct amdgpu_debugfs_regs2_data *rd = f->private_data;
 233         struct amdgpu_device *adev = rd->adev;
 234         ssize_t result = 0;
 235         int r;
 236         uint32_t value;
 237
 238         if (size & 0x3 || offset & 0x3)
 239                 return -EINVAL;
 240
 241         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 242         if (r < 0) {
 243                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 244                 return r;
 245         }
 246
 247         r = amdgpu_virt_enable_access_debugfs(adev);
 248         if (r < 0) {
 249                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 250                 return r;
 251         }
 252
 253         mutex_lock(&rd->lock);
 254
 255         if (rd->id.use_grbm) {
 256                 if ((rd->id.grbm.sh != 0xFFFFFFFF && rd->id.grbm.sh >= adev->gfx.config.max_sh_per_se) ||
 257                     (rd->id.grbm.se != 0xFFFFFFFF && rd->id.grbm.se >= adev->gfx.config.max_shader_engines)) {
 258                         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 259                         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 260                         amdgpu_virt_disable_access_debugfs(adev);
 261                         mutex_unlock(&rd->lock);
 262                         return -EINVAL;
 263                 }
 264                 mutex_lock(&adev->grbm_idx_mutex);
 265                 amdgpu_gfx_select_se_sh(adev, rd->id.grbm.se,
 266                                                   rd->id.grbm.sh,
 267                                                   rd->id.grbm.instance, rd->id.xcc_id);
 268         }
 269
 270         if (rd->id.use_srbm) {
 271                 mutex_lock(&adev->srbm_mutex);
 272                 amdgpu_gfx_select_me_pipe_q(adev, rd->id.srbm.me, rd->id.srbm.pipe,
 273                                             rd->id.srbm.queue, rd->id.srbm.vmid, rd->id.xcc_id);
 274         }
 275
 276         if (rd->id.pg_lock)
 277                 mutex_lock(&adev->pm.mutex);
 278
 279         while (size) {
 280                 if (!write_en) {
 281                         value = RREG32(offset >> 2);
 282                         r = put_user(value, (uint32_t *)buf);
 283                 } else {
 284                         r = get_user(value, (uint32_t *)buf);
 285                         if (!r)
 286                                 amdgpu_mm_wreg_mmio_rlc(adev, offset >> 2, value);
 287                 }
 288                 if (r) {
 289                         result = r;
 290                         goto end;
 291                 }
 292                 offset += 4;
 293                 size -= 4;
 294                 result += 4;
 295                 buf += 4;
 296         }
 297 end:
 298         if (rd->id.use_grbm) {
 299                 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, rd->id.xcc_id);
 300                 mutex_unlock(&adev->grbm_idx_mutex);
 301         }
 302
 303         if (rd->id.use_srbm) {
 304                 amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, rd->id.xcc_id);
 305                 mutex_unlock(&adev->srbm_mutex);
 306         }
 307
 308         if (rd->id.pg_lock)
 309                 mutex_unlock(&adev->pm.mutex);
 310
 311         mutex_unlock(&rd->lock);
 312
 313         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 314         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 315
 316         amdgpu_virt_disable_access_debugfs(adev);
 317         return result;
 318 }
 319
 320 static long amdgpu_debugfs_regs2_ioctl(struct file *f, unsigned int cmd, unsigned long data)
 321 {
 322         struct amdgpu_debugfs_regs2_data *rd = f->private_data;
 323         struct amdgpu_debugfs_regs2_iocdata v1_data;
 324         int r;
 325
 326         mutex_lock(&rd->lock);
 327
 328         switch (cmd) {
 329         case AMDGPU_DEBUGFS_REGS2_IOC_SET_STATE_V2:
 330                 r = copy_from_user(&rd->id, (struct amdgpu_debugfs_regs2_iocdata_v2 *)data,
 331                                    sizeof(rd->id));
 332                 if (r)
 333                         r = -EINVAL;
 334                 goto done;
 335         case AMDGPU_DEBUGFS_REGS2_IOC_SET_STATE:
 336                 r = copy_from_user(&v1_data, (struct amdgpu_debugfs_regs2_iocdata *)data,
 337                                    sizeof(v1_data));
 338                 if (r) {
 339                         r = -EINVAL;
 340                         goto done;
 341                 }
 342                 goto v1_copy;
 343         default:
 344                 r = -EINVAL;
 345                 goto done;
 346         }
 347
 348 v1_copy:
 349         rd->id.use_srbm = v1_data.use_srbm;
 350         rd->id.use_grbm = v1_data.use_grbm;
 351         rd->id.pg_lock = v1_data.pg_lock;
 352         rd->id.grbm.se = v1_data.grbm.se;
 353         rd->id.grbm.sh = v1_data.grbm.sh;
 354         rd->id.grbm.instance = v1_data.grbm.instance;
 355         rd->id.srbm.me = v1_data.srbm.me;
 356         rd->id.srbm.pipe = v1_data.srbm.pipe;
 357         rd->id.srbm.queue = v1_data.srbm.queue;
 358         rd->id.xcc_id = 0;
 359 done:
 360         mutex_unlock(&rd->lock);
 361         return r;
 362 }
 363
 364 static ssize_t amdgpu_debugfs_regs2_read(struct file *f, char __user *buf, size_t size, loff_t *pos)
 365 {
 366         return amdgpu_debugfs_regs2_op(f, buf, *pos, size, 0);
 367 }
 368
 369 static ssize_t amdgpu_debugfs_regs2_write(struct file *f, const char __user *buf, size_t size, loff_t *pos)
 370 {
 371         return amdgpu_debugfs_regs2_op(f, (char __user *)buf, *pos, size, 1);
 372 }
 373
 374 static int amdgpu_debugfs_gprwave_open(struct inode *inode, struct file *file)
 375 {
 376         struct amdgpu_debugfs_gprwave_data *rd;
 377
 378         rd = kzalloc(sizeof *rd, GFP_KERNEL);
 379         if (!rd)
 380                 return -ENOMEM;
 381         rd->adev = file_inode(file)->i_private;
 382         file->private_data = rd;
 383         mutex_init(&rd->lock);
 384
 385         return 0;
 386 }
 387
 388 static int amdgpu_debugfs_gprwave_release(struct inode *inode, struct file *file)
 389 {
 390         struct amdgpu_debugfs_gprwave_data *rd = file->private_data;
 391         mutex_destroy(&rd->lock);
 392         kfree(file->private_data);
 393         return 0;
 394 }
 395
 396 static ssize_t amdgpu_debugfs_gprwave_read(struct file *f, char __user *buf, size_t size, loff_t *pos)
 397 {
 398         struct amdgpu_debugfs_gprwave_data *rd = f->private_data;
 399         struct amdgpu_device *adev = rd->adev;
 400         ssize_t result = 0;
 401         int r;
 402         uint32_t *data, x;
 403
 404         if (size & 0x3 || *pos & 0x3)
 405                 return -EINVAL;
 406
 407         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 408         if (r < 0) {
 409                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 410                 return r;
 411         }
 412
 413         r = amdgpu_virt_enable_access_debugfs(adev);
 414         if (r < 0) {
 415                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 416                 return r;
 417         }
 418
 419         data = kcalloc(1024, sizeof(*data), GFP_KERNEL);
 420         if (!data) {
 421                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 422                 amdgpu_virt_disable_access_debugfs(adev);
 423                 return -ENOMEM;
 424         }
 425
 426         /* switch to the specific se/sh/cu */
 427         mutex_lock(&adev->grbm_idx_mutex);
 428         amdgpu_gfx_select_se_sh(adev, rd->id.se, rd->id.sh, rd->id.cu, rd->id.xcc_id);
 429
 430         if (!rd->id.gpr_or_wave) {
 431                 x = 0;
 432                 if (adev->gfx.funcs->read_wave_data)
 433                         adev->gfx.funcs->read_wave_data(adev, rd->id.xcc_id, rd->id.simd, rd->id.wave, data, &x);
 434         } else {
 435                 x = size >> 2;
 436                 if (rd->id.gpr.vpgr_or_sgpr) {
 437                         if (adev->gfx.funcs->read_wave_vgprs)
 438                                 adev->gfx.funcs->read_wave_vgprs(adev, rd->id.xcc_id, rd->id.simd, rd->id.wave, rd->id.gpr.thread, *pos, size>>2, data);
 439                 } else {
 440                         if (adev->gfx.funcs->read_wave_sgprs)
 441                                 adev->gfx.funcs->read_wave_sgprs(adev, rd->id.xcc_id, rd->id.simd, rd->id.wave, *pos, size>>2, data);
 442                 }
 443         }
 444
 445         amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, rd->id.xcc_id);
 446         mutex_unlock(&adev->grbm_idx_mutex);
 447
 448         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 449         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 450
 451         if (!x) {
 452                 result = -EINVAL;
 453                 goto done;
 454         }
 455
 456         while (size && (*pos < x * 4)) {
 457                 uint32_t value;
 458
 459                 value = data[*pos >> 2];
 460                 r = put_user(value, (uint32_t *)buf);
 461                 if (r) {
 462                         result = r;
 463                         goto done;
 464                 }
 465
 466                 result += 4;
 467                 buf += 4;
 468                 *pos += 4;
 469                 size -= 4;
 470         }
 471
 472 done:
 473         amdgpu_virt_disable_access_debugfs(adev);
 474         kfree(data);
 475         return result;
 476 }
 477
 478 static long amdgpu_debugfs_gprwave_ioctl(struct file *f, unsigned int cmd, unsigned long data)
 479 {
 480         struct amdgpu_debugfs_gprwave_data *rd = f->private_data;
 481         int r;
 482
 483         mutex_lock(&rd->lock);
 484
 485         switch (cmd) {
 486         case AMDGPU_DEBUGFS_GPRWAVE_IOC_SET_STATE:
 487                 r = copy_from_user(&rd->id, (struct amdgpu_debugfs_gprwave_iocdata *)data, sizeof rd->id);
 488                 if (r)
 489                         return r ? -EINVAL : 0;
 490                 goto done;
 491         default:
 492                 r = -EINVAL;
 493                 goto done;
 494         }
 495
 496 done:
 497         mutex_unlock(&rd->lock);
 498         return r;
 499 }
 500
 501
 502
 503
 504 /**
 505  * amdgpu_debugfs_regs_pcie_read - Read from a PCIE register
 506  *
 507  * @f: open file handle
 508  * @buf: User buffer to store read data in
 509  * @size: Number of bytes to read
 510  * @pos:  Offset to seek to
 511  *
 512  * The lower bits are the BYTE offset of the register to read.  This
 513  * allows reading multiple registers in a single call and having
 514  * the returned size reflect that.
 515  */
 516 static ssize_t amdgpu_debugfs_regs_pcie_read(struct file *f, char __user *buf,
 517                                         size_t size, loff_t *pos)
 518 {
 519         struct amdgpu_device *adev = file_inode(f)->i_private;
 520         ssize_t result = 0;
 521         int r;
 522
 523         if (size & 0x3 || *pos & 0x3)
 524                 return -EINVAL;
 525
 526         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 527         if (r < 0) {
 528                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 529                 return r;
 530         }
 531
 532         r = amdgpu_virt_enable_access_debugfs(adev);
 533         if (r < 0) {
 534                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 535                 return r;
 536         }
 537
 538         while (size) {
 539                 uint32_t value;
 540
 541                 value = RREG32_PCIE(*pos);
 542                 r = put_user(value, (uint32_t *)buf);
 543                 if (r)
 544                         goto out;
 545
 546                 result += 4;
 547                 buf += 4;
 548                 *pos += 4;
 549                 size -= 4;
 550         }
 551
 552         r = result;
 553 out:
 554         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 555         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 556         amdgpu_virt_disable_access_debugfs(adev);
 557         return r;
 558 }
 559
 560 /**
 561  * amdgpu_debugfs_regs_pcie_write - Write to a PCIE register
 562  *
 563  * @f: open file handle
 564  * @buf: User buffer to write data from
 565  * @size: Number of bytes to write
 566  * @pos:  Offset to seek to
 567  *
 568  * The lower bits are the BYTE offset of the register to write.  This
 569  * allows writing multiple registers in a single call and having
 570  * the returned size reflect that.
 571  */
 572 static ssize_t amdgpu_debugfs_regs_pcie_write(struct file *f, const char __user *buf,
 573                                          size_t size, loff_t *pos)
 574 {
 575         struct amdgpu_device *adev = file_inode(f)->i_private;
 576         ssize_t result = 0;
 577         int r;
 578
 579         if (size & 0x3 || *pos & 0x3)
 580                 return -EINVAL;
 581
 582         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 583         if (r < 0) {
 584                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 585                 return r;
 586         }
 587
 588         r = amdgpu_virt_enable_access_debugfs(adev);
 589         if (r < 0) {
 590                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 591                 return r;
 592         }
 593
 594         while (size) {
 595                 uint32_t value;
 596
 597                 r = get_user(value, (uint32_t *)buf);
 598                 if (r)
 599                         goto out;
 600
 601                 WREG32_PCIE(*pos, value);
 602
 603                 result += 4;
 604                 buf += 4;
 605                 *pos += 4;
 606                 size -= 4;
 607         }
 608
 609         r = result;
 610 out:
 611         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 612         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 613         amdgpu_virt_disable_access_debugfs(adev);
 614         return r;
 615 }
 616
 617 /**
 618  * amdgpu_debugfs_regs_didt_read - Read from a DIDT register
 619  *
 620  * @f: open file handle
 621  * @buf: User buffer to store read data in
 622  * @size: Number of bytes to read
 623  * @pos:  Offset to seek to
 624  *
 625  * The lower bits are the BYTE offset of the register to read.  This
 626  * allows reading multiple registers in a single call and having
 627  * the returned size reflect that.
 628  */
 629 static ssize_t amdgpu_debugfs_regs_didt_read(struct file *f, char __user *buf,
 630                                         size_t size, loff_t *pos)
 631 {
 632         struct amdgpu_device *adev = file_inode(f)->i_private;
 633         ssize_t result = 0;
 634         int r;
 635
 636         if (size & 0x3 || *pos & 0x3)
 637                 return -EINVAL;
 638
 639         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 640         if (r < 0) {
 641                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 642                 return r;
 643         }
 644
 645         r = amdgpu_virt_enable_access_debugfs(adev);
 646         if (r < 0) {
 647                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 648                 return r;
 649         }
 650
 651         while (size) {
 652                 uint32_t value;
 653
 654                 value = RREG32_DIDT(*pos >> 2);
 655                 r = put_user(value, (uint32_t *)buf);
 656                 if (r)
 657                         goto out;
 658
 659                 result += 4;
 660                 buf += 4;
 661                 *pos += 4;
 662                 size -= 4;
 663         }
 664
 665         r = result;
 666 out:
 667         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 668         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 669         amdgpu_virt_disable_access_debugfs(adev);
 670         return r;
 671 }
 672
 673 /**
 674  * amdgpu_debugfs_regs_didt_write - Write to a DIDT register
 675  *
 676  * @f: open file handle
 677  * @buf: User buffer to write data from
 678  * @size: Number of bytes to write
 679  * @pos:  Offset to seek to
 680  *
 681  * The lower bits are the BYTE offset of the register to write.  This
 682  * allows writing multiple registers in a single call and having
 683  * the returned size reflect that.
 684  */
 685 static ssize_t amdgpu_debugfs_regs_didt_write(struct file *f, const char __user *buf,
 686                                          size_t size, loff_t *pos)
 687 {
 688         struct amdgpu_device *adev = file_inode(f)->i_private;
 689         ssize_t result = 0;
 690         int r;
 691
 692         if (size & 0x3 || *pos & 0x3)
 693                 return -EINVAL;
 694
 695         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 696         if (r < 0) {
 697                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 698                 return r;
 699         }
 700
 701         r = amdgpu_virt_enable_access_debugfs(adev);
 702         if (r < 0) {
 703                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 704                 return r;
 705         }
 706
 707         while (size) {
 708                 uint32_t value;
 709
 710                 r = get_user(value, (uint32_t *)buf);
 711                 if (r)
 712                         goto out;
 713
 714                 WREG32_DIDT(*pos >> 2, value);
 715
 716                 result += 4;
 717                 buf += 4;
 718                 *pos += 4;
 719                 size -= 4;
 720         }
 721
 722         r = result;
 723 out:
 724         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 725         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 726         amdgpu_virt_disable_access_debugfs(adev);
 727         return r;
 728 }
 729
 730 /**
 731  * amdgpu_debugfs_regs_smc_read - Read from a SMC register
 732  *
 733  * @f: open file handle
 734  * @buf: User buffer to store read data in
 735  * @size: Number of bytes to read
 736  * @pos:  Offset to seek to
 737  *
 738  * The lower bits are the BYTE offset of the register to read.  This
 739  * allows reading multiple registers in a single call and having
 740  * the returned size reflect that.
 741  */
 742 static ssize_t amdgpu_debugfs_regs_smc_read(struct file *f, char __user *buf,
 743                                         size_t size, loff_t *pos)
 744 {
 745         struct amdgpu_device *adev = file_inode(f)->i_private;
 746         ssize_t result = 0;
 747         int r;
 748
 749         if (size & 0x3 || *pos & 0x3)
 750                 return -EINVAL;
 751
 752         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 753         if (r < 0) {
 754                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 755                 return r;
 756         }
 757
 758         r = amdgpu_virt_enable_access_debugfs(adev);
 759         if (r < 0) {
 760                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 761                 return r;
 762         }
 763
 764         while (size) {
 765                 uint32_t value;
 766
 767                 value = RREG32_SMC(*pos);
 768                 r = put_user(value, (uint32_t *)buf);
 769                 if (r)
 770                         goto out;
 771
 772                 result += 4;
 773                 buf += 4;
 774                 *pos += 4;
 775                 size -= 4;
 776         }
 777
 778         r = result;
 779 out:
 780         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 781         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 782         amdgpu_virt_disable_access_debugfs(adev);
 783         return r;
 784 }
 785
 786 /**
 787  * amdgpu_debugfs_regs_smc_write - Write to a SMC register
 788  *
 789  * @f: open file handle
 790  * @buf: User buffer to write data from
 791  * @size: Number of bytes to write
 792  * @pos:  Offset to seek to
 793  *
 794  * The lower bits are the BYTE offset of the register to write.  This
 795  * allows writing multiple registers in a single call and having
 796  * the returned size reflect that.
 797  */
 798 static ssize_t amdgpu_debugfs_regs_smc_write(struct file *f, const char __user *buf,
 799                                          size_t size, loff_t *pos)
 800 {
 801         struct amdgpu_device *adev = file_inode(f)->i_private;
 802         ssize_t result = 0;
 803         int r;
 804
 805         if (size & 0x3 || *pos & 0x3)
 806                 return -EINVAL;
 807
 808         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 809         if (r < 0) {
 810                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 811                 return r;
 812         }
 813
 814         r = amdgpu_virt_enable_access_debugfs(adev);
 815         if (r < 0) {
 816                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 817                 return r;
 818         }
 819
 820         while (size) {
 821                 uint32_t value;
 822
 823                 r = get_user(value, (uint32_t *)buf);
 824                 if (r)
 825                         goto out;
 826
 827                 WREG32_SMC(*pos, value);
 828
 829                 result += 4;
 830                 buf += 4;
 831                 *pos += 4;
 832                 size -= 4;
 833         }
 834
 835         r = result;
 836 out:
 837         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 838         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 839         amdgpu_virt_disable_access_debugfs(adev);
 840         return r;
 841 }
 842
 843 /**
 844  * amdgpu_debugfs_gca_config_read - Read from gfx config data
 845  *
 846  * @f: open file handle
 847  * @buf: User buffer to store read data in
 848  * @size: Number of bytes to read
 849  * @pos:  Offset to seek to
 850  *
 851  * This file is used to access configuration data in a somewhat
 852  * stable fashion.  The format is a series of DWORDs with the first
 853  * indicating which revision it is.  New content is appended to the
 854  * end so that older software can still read the data.
 855  */
 856
 857 static ssize_t amdgpu_debugfs_gca_config_read(struct file *f, char __user *buf,
 858                                         size_t size, loff_t *pos)
 859 {
 860         struct amdgpu_device *adev = file_inode(f)->i_private;
 861         ssize_t result = 0;
 862         int r;
 863         uint32_t *config, no_regs = 0;
 864
 865         if (size & 0x3 || *pos & 0x3)
 866                 return -EINVAL;
 867
 868         config = kmalloc_array(256, sizeof(*config), GFP_KERNEL);
 869         if (!config)
 870                 return -ENOMEM;
 871
 872         /* version, increment each time something is added */
 873         config[no_regs++] = 5;
 874         config[no_regs++] = adev->gfx.config.max_shader_engines;
 875         config[no_regs++] = adev->gfx.config.max_tile_pipes;
 876         config[no_regs++] = adev->gfx.config.max_cu_per_sh;
 877         config[no_regs++] = adev->gfx.config.max_sh_per_se;
 878         config[no_regs++] = adev->gfx.config.max_backends_per_se;
 879         config[no_regs++] = adev->gfx.config.max_texture_channel_caches;
 880         config[no_regs++] = adev->gfx.config.max_gprs;
 881         config[no_regs++] = adev->gfx.config.max_gs_threads;
 882         config[no_regs++] = adev->gfx.config.max_hw_contexts;
 883         config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_frontend;
 884         config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_backend;
 885         config[no_regs++] = adev->gfx.config.sc_hiz_tile_fifo_size;
 886         config[no_regs++] = adev->gfx.config.sc_earlyz_tile_fifo_size;
 887         config[no_regs++] = adev->gfx.config.num_tile_pipes;
 888         config[no_regs++] = adev->gfx.config.backend_enable_mask;
 889         config[no_regs++] = adev->gfx.config.mem_max_burst_length_bytes;
 890         config[no_regs++] = adev->gfx.config.mem_row_size_in_kb;
 891         config[no_regs++] = adev->gfx.config.shader_engine_tile_size;
 892         config[no_regs++] = adev->gfx.config.num_gpus;
 893         config[no_regs++] = adev->gfx.config.multi_gpu_tile_size;
 894         config[no_regs++] = adev->gfx.config.mc_arb_ramcfg;
 895         config[no_regs++] = adev->gfx.config.gb_addr_config;
 896         config[no_regs++] = adev->gfx.config.num_rbs;
 897
 898         /* rev==1 */
 899         config[no_regs++] = adev->rev_id;
 900         config[no_regs++] = lower_32_bits(adev->pg_flags);
 901         config[no_regs++] = lower_32_bits(adev->cg_flags);
 902
 903         /* rev==2 */
 904         config[no_regs++] = adev->family;
 905         config[no_regs++] = adev->external_rev_id;
 906
 907         /* rev==3 */
 908         config[no_regs++] = adev->pdev->device;
 909         config[no_regs++] = adev->pdev->revision;
 910         config[no_regs++] = adev->pdev->subsystem_device;
 911         config[no_regs++] = adev->pdev->subsystem_vendor;
 912
 913         /* rev==4 APU flag */
 914         config[no_regs++] = adev->flags & AMD_IS_APU ? 1 : 0;
 915
 916         /* rev==5 PG/CG flag upper 32bit */
 917         config[no_regs++] = upper_32_bits(adev->pg_flags);
 918         config[no_regs++] = upper_32_bits(adev->cg_flags);
 919
 920         while (size && (*pos < no_regs * 4)) {
 921                 uint32_t value;
 922
 923                 value = config[*pos >> 2];
 924                 r = put_user(value, (uint32_t *)buf);
 925                 if (r) {
 926                         kfree(config);
 927                         return r;
 928                 }
 929
 930                 result += 4;
 931                 buf += 4;
 932                 *pos += 4;
 933                 size -= 4;
 934         }
 935
 936         kfree(config);
 937         return result;
 938 }
 939
 940 /**
 941  * amdgpu_debugfs_sensor_read - Read from the powerplay sensors
 942  *
 943  * @f: open file handle
 944  * @buf: User buffer to store read data in
 945  * @size: Number of bytes to read
 946  * @pos:  Offset to seek to
 947  *
 948  * The offset is treated as the BYTE address of one of the sensors
 949  * enumerated in amd/include/kgd_pp_interface.h under the
 950  * 'amd_pp_sensors' enumeration.  For instance to read the UVD VCLK
 951  * you would use the offset 3 * 4 = 12.
 952  */
 953 static ssize_t amdgpu_debugfs_sensor_read(struct file *f, char __user *buf,
 954                                         size_t size, loff_t *pos)
 955 {
 956         struct amdgpu_device *adev = file_inode(f)->i_private;
 957         int idx, x, outsize, r, valuesize;
 958         uint32_t values[16];
 959
 960         if (size & 3 || *pos & 0x3)
 961                 return -EINVAL;
 962
 963         if (!adev->pm.dpm_enabled)
 964                 return -EINVAL;
 965
 966         /* convert offset to sensor number */
 967         idx = *pos >> 2;
 968
 969         valuesize = sizeof(values);
 970
 971         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 972         if (r < 0) {
 973                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 974                 return r;
 975         }
 976
 977         r = amdgpu_virt_enable_access_debugfs(adev);
 978         if (r < 0) {
 979                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 980                 return r;
 981         }
 982
 983         r = amdgpu_dpm_read_sensor(adev, idx, &values[0], &valuesize);
 984
 985         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 986         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 987
 988         if (r) {
 989                 amdgpu_virt_disable_access_debugfs(adev);
 990                 return r;
 991         }
 992
 993         if (size > valuesize) {
 994                 amdgpu_virt_disable_access_debugfs(adev);
 995                 return -EINVAL;
 996         }
 997
 998         outsize = 0;
 999         x = 0;
1000         if (!r) {
1001                 while (size) {
1002                         r = put_user(values[x++], (int32_t *)buf);
1003                         buf += 4;
1004                         size -= 4;
1005                         outsize += 4;
1006                 }
1007         }
1008
1009         amdgpu_virt_disable_access_debugfs(adev);
1010         return !r ? outsize : r;
1011 }
1012
1013 /** amdgpu_debugfs_wave_read - Read WAVE STATUS data
1014  *
1015  * @f: open file handle
1016  * @buf: User buffer to store read data in
1017  * @size: Number of bytes to read
1018  * @pos:  Offset to seek to
1019  *
1020  * The offset being sought changes which wave that the status data
1021  * will be returned for.  The bits are used as follows:
1022  *
1023  * Bits 0..6:   Byte offset into data
1024  * Bits 7..14:  SE selector
1025  * Bits 15..22: SH/SA selector
1026  * Bits 23..30: CU/{WGP+SIMD} selector
1027  * Bits 31..36: WAVE ID selector
1028  * Bits 37..44: SIMD ID selector
1029  *
1030  * The returned data begins with one DWORD of version information
1031  * Followed by WAVE STATUS registers relevant to the GFX IP version
1032  * being used.  See gfx_v8_0_read_wave_data() for an example output.
1033  */
1034 static ssize_t amdgpu_debugfs_wave_read(struct file *f, char __user *buf,
1035                                         size_t size, loff_t *pos)
1036 {
1037         struct amdgpu_device *adev = f->f_inode->i_private;
1038         int r, x;
1039         ssize_t result = 0;
1040         uint32_t offset, se, sh, cu, wave, simd, data[32];
1041
1042         if (size & 3 || *pos & 3)
1043                 return -EINVAL;
1044
1045         /* decode offset */
1046         offset = (*pos & GENMASK_ULL(6, 0));
1047         se = (*pos & GENMASK_ULL(14, 7)) >> 7;
1048         sh = (*pos & GENMASK_ULL(22, 15)) >> 15;
1049         cu = (*pos & GENMASK_ULL(30, 23)) >> 23;
1050         wave = (*pos & GENMASK_ULL(36, 31)) >> 31;
1051         simd = (*pos & GENMASK_ULL(44, 37)) >> 37;
1052
1053         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1054         if (r < 0) {
1055                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1056                 return r;
1057         }
1058
1059         r = amdgpu_virt_enable_access_debugfs(adev);
1060         if (r < 0) {
1061                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1062                 return r;
1063         }
1064
1065         /* switch to the specific se/sh/cu */
1066         mutex_lock(&adev->grbm_idx_mutex);
1067         amdgpu_gfx_select_se_sh(adev, se, sh, cu, 0);
1068
1069         x = 0;
1070         if (adev->gfx.funcs->read_wave_data)
1071                 adev->gfx.funcs->read_wave_data(adev, 0, simd, wave, data, &x);
1072
1073         amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0);
1074         mutex_unlock(&adev->grbm_idx_mutex);
1075
1076         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1077         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1078
1079         if (!x) {
1080                 amdgpu_virt_disable_access_debugfs(adev);
1081                 return -EINVAL;
1082         }
1083
1084         while (size && (offset < x * 4)) {
1085                 uint32_t value;
1086
1087                 value = data[offset >> 2];
1088                 r = put_user(value, (uint32_t *)buf);
1089                 if (r) {
1090                         amdgpu_virt_disable_access_debugfs(adev);
1091                         return r;
1092                 }
1093
1094                 result += 4;
1095                 buf += 4;
1096                 offset += 4;
1097                 size -= 4;
1098         }
1099
1100         amdgpu_virt_disable_access_debugfs(adev);
1101         return result;
1102 }
1103
1104 /** amdgpu_debugfs_gpr_read - Read wave gprs
1105  *
1106  * @f: open file handle
1107  * @buf: User buffer to store read data in
1108  * @size: Number of bytes to read
1109  * @pos:  Offset to seek to
1110  *
1111  * The offset being sought changes which wave that the status data
1112  * will be returned for.  The bits are used as follows:
1113  *
1114  * Bits 0..11:  Byte offset into data
1115  * Bits 12..19: SE selector
1116  * Bits 20..27: SH/SA selector
1117  * Bits 28..35: CU/{WGP+SIMD} selector
1118  * Bits 36..43: WAVE ID selector
1119  * Bits 37..44: SIMD ID selector
1120  * Bits 52..59: Thread selector
1121  * Bits 60..61: Bank selector (VGPR=0,SGPR=1)
1122  *
1123  * The return data comes from the SGPR or VGPR register bank for
1124  * the selected operational unit.
1125  */
1126 static ssize_t amdgpu_debugfs_gpr_read(struct file *f, char __user *buf,
1127                                         size_t size, loff_t *pos)
1128 {
1129         struct amdgpu_device *adev = f->f_inode->i_private;
1130         int r;
1131         ssize_t result = 0;
1132         uint32_t offset, se, sh, cu, wave, simd, thread, bank, *data;
1133
1134         if (size > 4096 || size & 3 || *pos & 3)
1135                 return -EINVAL;
1136
1137         /* decode offset */
1138         offset = (*pos & GENMASK_ULL(11, 0)) >> 2;
1139         se = (*pos & GENMASK_ULL(19, 12)) >> 12;
1140         sh = (*pos & GENMASK_ULL(27, 20)) >> 20;
1141         cu = (*pos & GENMASK_ULL(35, 28)) >> 28;
1142         wave = (*pos & GENMASK_ULL(43, 36)) >> 36;
1143         simd = (*pos & GENMASK_ULL(51, 44)) >> 44;
1144         thread = (*pos & GENMASK_ULL(59, 52)) >> 52;
1145         bank = (*pos & GENMASK_ULL(61, 60)) >> 60;
1146
1147         data = kcalloc(1024, sizeof(*data), GFP_KERNEL);
1148         if (!data)
1149                 return -ENOMEM;
1150
1151         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1152         if (r < 0)
1153                 goto err;
1154
1155         r = amdgpu_virt_enable_access_debugfs(adev);
1156         if (r < 0)
1157                 goto err;
1158
1159         /* switch to the specific se/sh/cu */
1160         mutex_lock(&adev->grbm_idx_mutex);
1161         amdgpu_gfx_select_se_sh(adev, se, sh, cu, 0);
1162
1163         if (bank == 0) {
1164                 if (adev->gfx.funcs->read_wave_vgprs)
1165                         adev->gfx.funcs->read_wave_vgprs(adev, 0, simd, wave, thread, offset, size>>2, data);
1166         } else {
1167                 if (adev->gfx.funcs->read_wave_sgprs)
1168                         adev->gfx.funcs->read_wave_sgprs(adev, 0, simd, wave, offset, size>>2, data);
1169         }
1170
1171         amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0);
1172         mutex_unlock(&adev->grbm_idx_mutex);
1173
1174         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1175         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1176
1177         while (size) {
1178                 uint32_t value;
1179
1180                 value = data[result >> 2];
1181                 r = put_user(value, (uint32_t *)buf);
1182                 if (r) {
1183                         amdgpu_virt_disable_access_debugfs(adev);
1184                         goto err;
1185                 }
1186
1187                 result += 4;
1188                 buf += 4;
1189                 size -= 4;
1190         }
1191
1192         kfree(data);
1193         amdgpu_virt_disable_access_debugfs(adev);
1194         return result;
1195
1196 err:
1197         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1198         kfree(data);
1199         return r;
1200 }
1201
1202 /**
1203  * amdgpu_debugfs_gfxoff_residency_read - Read GFXOFF residency
1204  *
1205  * @f: open file handle
1206  * @buf: User buffer to store read data in
1207  * @size: Number of bytes to read
1208  * @pos:  Offset to seek to
1209  *
1210  * Read the last residency value logged. It doesn't auto update, one needs to
1211  * stop logging before getting the current value.
1212  */
1213 static ssize_t amdgpu_debugfs_gfxoff_residency_read(struct file *f, char __user *buf,
1214                                                     size_t size, loff_t *pos)
1215 {
1216         struct amdgpu_device *adev = file_inode(f)->i_private;
1217         ssize_t result = 0;
1218         int r;
1219
1220         if (size & 0x3 || *pos & 0x3)
1221                 return -EINVAL;
1222
1223         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1224         if (r < 0) {
1225                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1226                 return r;
1227         }
1228
1229         while (size) {
1230                 uint32_t value;
1231
1232                 r = amdgpu_get_gfx_off_residency(adev, &value);
1233                 if (r)
1234                         goto out;
1235
1236                 r = put_user(value, (uint32_t *)buf);
1237                 if (r)
1238                         goto out;
1239
1240                 result += 4;
1241                 buf += 4;
1242                 *pos += 4;
1243                 size -= 4;
1244         }
1245
1246         r = result;
1247 out:
1248         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1249         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1250
1251         return r;
1252 }
1253
1254 /**
1255  * amdgpu_debugfs_gfxoff_residency_write - Log GFXOFF Residency
1256  *
1257  * @f: open file handle
1258  * @buf: User buffer to write data from
1259  * @size: Number of bytes to write
1260  * @pos:  Offset to seek to
1261  *
1262  * Write a 32-bit non-zero to start logging; write a 32-bit zero to stop
1263  */
1264 static ssize_t amdgpu_debugfs_gfxoff_residency_write(struct file *f, const char __user *buf,
1265                                                      size_t size, loff_t *pos)
1266 {
1267         struct amdgpu_device *adev = file_inode(f)->i_private;
1268         ssize_t result = 0;
1269         int r;
1270
1271         if (size & 0x3 || *pos & 0x3)
1272                 return -EINVAL;
1273
1274         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1275         if (r < 0) {
1276                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1277                 return r;
1278         }
1279
1280         while (size) {
1281                 u32 value;
1282
1283                 r = get_user(value, (uint32_t *)buf);
1284                 if (r)
1285                         goto out;
1286
1287                 amdgpu_set_gfx_off_residency(adev, value ? true : false);
1288
1289                 result += 4;
1290                 buf += 4;
1291                 *pos += 4;
1292                 size -= 4;
1293         }
1294
1295         r = result;
1296 out:
1297         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1298         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1299
1300         return r;
1301 }
1302
1303
1304 /**
1305  * amdgpu_debugfs_gfxoff_count_read - Read GFXOFF entry count
1306  *
1307  * @f: open file handle
1308  * @buf: User buffer to store read data in
1309  * @size: Number of bytes to read
1310  * @pos:  Offset to seek to
1311  */
1312 static ssize_t amdgpu_debugfs_gfxoff_count_read(struct file *f, char __user *buf,
1313                                                 size_t size, loff_t *pos)
1314 {
1315         struct amdgpu_device *adev = file_inode(f)->i_private;
1316         ssize_t result = 0;
1317         int r;
1318
1319         if (size & 0x3 || *pos & 0x3)
1320                 return -EINVAL;
1321
1322         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1323         if (r < 0) {
1324                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1325                 return r;
1326         }
1327
1328         while (size) {
1329                 u64 value = 0;
1330
1331                 r = amdgpu_get_gfx_off_entrycount(adev, &value);
1332                 if (r)
1333                         goto out;
1334
1335                 r = put_user(value, (u64 *)buf);
1336                 if (r)
1337                         goto out;
1338
1339                 result += 4;
1340                 buf += 4;
1341                 *pos += 4;
1342                 size -= 4;
1343         }
1344
1345         r = result;
1346 out:
1347         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1348         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1349
1350         return r;
1351 }
1352
1353 /**
1354  * amdgpu_debugfs_gfxoff_write - Enable/disable GFXOFF
1355  *
1356  * @f: open file handle
1357  * @buf: User buffer to write data from
1358  * @size: Number of bytes to write
1359  * @pos:  Offset to seek to
1360  *
1361  * Write a 32-bit zero to disable or a 32-bit non-zero to enable
1362  */
1363 static ssize_t amdgpu_debugfs_gfxoff_write(struct file *f, const char __user *buf,
1364                                          size_t size, loff_t *pos)
1365 {
1366         struct amdgpu_device *adev = file_inode(f)->i_private;
1367         ssize_t result = 0;
1368         int r;
1369
1370         if (size & 0x3 || *pos & 0x3)
1371                 return -EINVAL;
1372
1373         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1374         if (r < 0) {
1375                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1376                 return r;
1377         }
1378
1379         while (size) {
1380                 uint32_t value;
1381
1382                 r = get_user(value, (uint32_t *)buf);
1383                 if (r)
1384                         goto out;
1385
1386                 amdgpu_gfx_off_ctrl(adev, value ? true : false);
1387
1388                 result += 4;
1389                 buf += 4;
1390                 *pos += 4;
1391                 size -= 4;
1392         }
1393
1394         r = result;
1395 out:
1396         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1397         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1398
1399         return r;
1400 }
1401
1402
1403 /**
1404  * amdgpu_debugfs_gfxoff_read - read gfxoff status
1405  *
1406  * @f: open file handle
1407  * @buf: User buffer to store read data in
1408  * @size: Number of bytes to read
1409  * @pos:  Offset to seek to
1410  */
1411 static ssize_t amdgpu_debugfs_gfxoff_read(struct file *f, char __user *buf,
1412                                          size_t size, loff_t *pos)
1413 {
1414         struct amdgpu_device *adev = file_inode(f)->i_private;
1415         ssize_t result = 0;
1416         int r;
1417
1418         if (size & 0x3 || *pos & 0x3)
1419                 return -EINVAL;
1420
1421         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1422         if (r < 0) {
1423                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1424                 return r;
1425         }
1426
1427         while (size) {
1428                 u32 value = adev->gfx.gfx_off_state;
1429
1430                 r = put_user(value, (u32 *)buf);
1431                 if (r)
1432                         goto out;
1433
1434                 result += 4;
1435                 buf += 4;
1436                 *pos += 4;
1437                 size -= 4;
1438         }
1439
1440         r = result;
1441 out:
1442         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1443         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1444
1445         return r;
1446 }
1447
1448 static ssize_t amdgpu_debugfs_gfxoff_status_read(struct file *f, char __user *buf,
1449                                                  size_t size, loff_t *pos)
1450 {
1451         struct amdgpu_device *adev = file_inode(f)->i_private;
1452         ssize_t result = 0;
1453         int r;
1454
1455         if (size & 0x3 || *pos & 0x3)
1456                 return -EINVAL;
1457
1458         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1459         if (r < 0) {
1460                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1461                 return r;
1462         }
1463
1464         while (size) {
1465                 u32 value;
1466
1467                 r = amdgpu_get_gfx_off_status(adev, &value);
1468                 if (r)
1469                         goto out;
1470
1471                 r = put_user(value, (u32 *)buf);
1472                 if (r)
1473                         goto out;
1474
1475                 result += 4;
1476                 buf += 4;
1477                 *pos += 4;
1478                 size -= 4;
1479         }
1480
1481         r = result;
1482 out:
1483         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1484         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1485
1486         return r;
1487 }
1488
1489 static const struct file_operations amdgpu_debugfs_regs2_fops = {
1490         .owner = THIS_MODULE,
1491         .unlocked_ioctl = amdgpu_debugfs_regs2_ioctl,
1492         .read = amdgpu_debugfs_regs2_read,
1493         .write = amdgpu_debugfs_regs2_write,
1494         .open = amdgpu_debugfs_regs2_open,
1495         .release = amdgpu_debugfs_regs2_release,
1496         .llseek = default_llseek
1497 };
1498
1499 static const struct file_operations amdgpu_debugfs_gprwave_fops = {
1500         .owner = THIS_MODULE,
1501         .unlocked_ioctl = amdgpu_debugfs_gprwave_ioctl,
1502         .read = amdgpu_debugfs_gprwave_read,
1503         .open = amdgpu_debugfs_gprwave_open,
1504         .release = amdgpu_debugfs_gprwave_release,
1505         .llseek = default_llseek
1506 };
1507
1508 static const struct file_operations amdgpu_debugfs_regs_fops = {
1509         .owner = THIS_MODULE,
1510         .read = amdgpu_debugfs_regs_read,
1511         .write = amdgpu_debugfs_regs_write,
1512         .llseek = default_llseek
1513 };
1514 static const struct file_operations amdgpu_debugfs_regs_didt_fops = {
1515         .owner = THIS_MODULE,
1516         .read = amdgpu_debugfs_regs_didt_read,
1517         .write = amdgpu_debugfs_regs_didt_write,
1518         .llseek = default_llseek
1519 };
1520 static const struct file_operations amdgpu_debugfs_regs_pcie_fops = {
1521         .owner = THIS_MODULE,
1522         .read = amdgpu_debugfs_regs_pcie_read,
1523         .write = amdgpu_debugfs_regs_pcie_write,
1524         .llseek = default_llseek
1525 };
1526 static const struct file_operations amdgpu_debugfs_regs_smc_fops = {
1527         .owner = THIS_MODULE,
1528         .read = amdgpu_debugfs_regs_smc_read,
1529         .write = amdgpu_debugfs_regs_smc_write,
1530         .llseek = default_llseek
1531 };
1532
1533 static const struct file_operations amdgpu_debugfs_gca_config_fops = {
1534         .owner = THIS_MODULE,
1535         .read = amdgpu_debugfs_gca_config_read,
1536         .llseek = default_llseek
1537 };
1538
1539 static const struct file_operations amdgpu_debugfs_sensors_fops = {
1540         .owner = THIS_MODULE,
1541         .read = amdgpu_debugfs_sensor_read,
1542         .llseek = default_llseek
1543 };
1544
1545 static const struct file_operations amdgpu_debugfs_wave_fops = {
1546         .owner = THIS_MODULE,
1547         .read = amdgpu_debugfs_wave_read,
1548         .llseek = default_llseek
1549 };
1550 static const struct file_operations amdgpu_debugfs_gpr_fops = {
1551         .owner = THIS_MODULE,
1552         .read = amdgpu_debugfs_gpr_read,
1553         .llseek = default_llseek
1554 };
1555
1556 static const struct file_operations amdgpu_debugfs_gfxoff_fops = {
1557         .owner = THIS_MODULE,
1558         .read = amdgpu_debugfs_gfxoff_read,
1559         .write = amdgpu_debugfs_gfxoff_write,
1560         .llseek = default_llseek
1561 };
1562
1563 static const struct file_operations amdgpu_debugfs_gfxoff_status_fops = {
1564         .owner = THIS_MODULE,
1565         .read = amdgpu_debugfs_gfxoff_status_read,
1566         .llseek = default_llseek
1567 };
1568
1569 static const struct file_operations amdgpu_debugfs_gfxoff_count_fops = {
1570         .owner = THIS_MODULE,
1571         .read = amdgpu_debugfs_gfxoff_count_read,
1572         .llseek = default_llseek
1573 };
1574
1575 static const struct file_operations amdgpu_debugfs_gfxoff_residency_fops = {
1576         .owner = THIS_MODULE,
1577         .read = amdgpu_debugfs_gfxoff_residency_read,
1578         .write = amdgpu_debugfs_gfxoff_residency_write,
1579         .llseek = default_llseek
1580 };
1581
1582 static const struct file_operations *debugfs_regs[] = {
1583         &amdgpu_debugfs_regs_fops,
1584         &amdgpu_debugfs_regs2_fops,
1585         &amdgpu_debugfs_gprwave_fops,
1586         &amdgpu_debugfs_regs_didt_fops,
1587         &amdgpu_debugfs_regs_pcie_fops,
1588         &amdgpu_debugfs_regs_smc_fops,
1589         &amdgpu_debugfs_gca_config_fops,
1590         &amdgpu_debugfs_sensors_fops,
1591         &amdgpu_debugfs_wave_fops,
1592         &amdgpu_debugfs_gpr_fops,
1593         &amdgpu_debugfs_gfxoff_fops,
1594         &amdgpu_debugfs_gfxoff_status_fops,
1595         &amdgpu_debugfs_gfxoff_count_fops,
1596         &amdgpu_debugfs_gfxoff_residency_fops,
1597 };
1598
1599 static const char * const debugfs_regs_names[] = {
1600         "amdgpu_regs",
1601         "amdgpu_regs2",
1602         "amdgpu_gprwave",
1603         "amdgpu_regs_didt",
1604         "amdgpu_regs_pcie",
1605         "amdgpu_regs_smc",
1606         "amdgpu_gca_config",
1607         "amdgpu_sensors",
1608         "amdgpu_wave",
1609         "amdgpu_gpr",
1610         "amdgpu_gfxoff",
1611         "amdgpu_gfxoff_status",
1612         "amdgpu_gfxoff_count",
1613         "amdgpu_gfxoff_residency",
1614 };
1615
1616 /**
1617  * amdgpu_debugfs_regs_init -   Initialize debugfs entries that provide
1618  *                              register access.
1619  *
1620  * @adev: The device to attach the debugfs entries to
1621  */
1622 int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
1623 {
1624         struct drm_minor *minor = adev_to_drm(adev)->primary;
1625         struct dentry *ent, *root = minor->debugfs_root;
1626         unsigned int i;
1627
1628         for (i = 0; i < ARRAY_SIZE(debugfs_regs); i++) {
1629                 ent = debugfs_create_file(debugfs_regs_names[i],
1630                                           S_IFREG | 0444, root,
1631                                           adev, debugfs_regs[i]);
1632                 if (!i && !IS_ERR_OR_NULL(ent))
1633                         i_size_write(ent->d_inode, adev->rmmio_size);
1634         }
1635
1636         return 0;
1637 }
1638
1639 static int amdgpu_debugfs_test_ib_show(struct seq_file *m, void *unused)
1640 {
1641         struct amdgpu_device *adev = m->private;
1642         struct drm_device *dev = adev_to_drm(adev);
1643         int r = 0, i;
1644
1645         r = pm_runtime_get_sync(dev->dev);
1646         if (r < 0) {
1647                 pm_runtime_put_autosuspend(dev->dev);
1648                 return r;
1649         }
1650
1651         /* Avoid accidently unparking the sched thread during GPU reset */
1652         r = down_write_killable(&adev->reset_domain->sem);
1653         if (r)
1654                 return r;
1655
1656         /* hold on the scheduler */
1657         for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
1658                 struct amdgpu_ring *ring = adev->rings[i];
1659
1660                 if (!ring || !ring->sched.thread)
1661                         continue;
1662                 kthread_park(ring->sched.thread);
1663         }
1664
1665         seq_puts(m, "run ib test:\n");
1666         r = amdgpu_ib_ring_tests(adev);
1667         if (r)
1668                 seq_printf(m, "ib ring tests failed (%d).\n", r);
1669         else
1670                 seq_puts(m, "ib ring tests passed.\n");
1671
1672         /* go on the scheduler */
1673         for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
1674                 struct amdgpu_ring *ring = adev->rings[i];
1675
1676                 if (!ring || !ring->sched.thread)
1677                         continue;
1678                 kthread_unpark(ring->sched.thread);
1679         }
1680
1681         up_write(&adev->reset_domain->sem);
1682
1683         pm_runtime_mark_last_busy(dev->dev);
1684         pm_runtime_put_autosuspend(dev->dev);
1685
1686         return 0;
1687 }
1688
1689 static int amdgpu_debugfs_evict_vram(void *data, u64 *val)
1690 {
1691         struct amdgpu_device *adev = (struct amdgpu_device *)data;
1692         struct drm_device *dev = adev_to_drm(adev);
1693         int r;
1694
1695         r = pm_runtime_get_sync(dev->dev);
1696         if (r < 0) {
1697                 pm_runtime_put_autosuspend(dev->dev);
1698                 return r;
1699         }
1700
1701         *val = amdgpu_ttm_evict_resources(adev, TTM_PL_VRAM);
1702
1703         pm_runtime_mark_last_busy(dev->dev);
1704         pm_runtime_put_autosuspend(dev->dev);
1705
1706         return 0;
1707 }
1708
1709
1710 static int amdgpu_debugfs_evict_gtt(void *data, u64 *val)
1711 {
1712         struct amdgpu_device *adev = (struct amdgpu_device *)data;
1713         struct drm_device *dev = adev_to_drm(adev);
1714         int r;
1715
1716         r = pm_runtime_get_sync(dev->dev);
1717         if (r < 0) {
1718                 pm_runtime_put_autosuspend(dev->dev);
1719                 return r;
1720         }
1721
1722         *val = amdgpu_ttm_evict_resources(adev, TTM_PL_TT);
1723
1724         pm_runtime_mark_last_busy(dev->dev);
1725         pm_runtime_put_autosuspend(dev->dev);
1726
1727         return 0;
1728 }
1729
1730 static int amdgpu_debugfs_benchmark(void *data, u64 val)
1731 {
1732         struct amdgpu_device *adev = (struct amdgpu_device *)data;
1733         struct drm_device *dev = adev_to_drm(adev);
1734         int r;
1735
1736         r = pm_runtime_get_sync(dev->dev);
1737         if (r < 0) {
1738                 pm_runtime_put_autosuspend(dev->dev);
1739                 return r;
1740         }
1741
1742         r = amdgpu_benchmark(adev, val);
1743
1744         pm_runtime_mark_last_busy(dev->dev);
1745         pm_runtime_put_autosuspend(dev->dev);
1746
1747         return r;
1748 }
1749
1750 static int amdgpu_debugfs_vm_info_show(struct seq_file *m, void *unused)
1751 {
1752         struct amdgpu_device *adev = m->private;
1753         struct drm_device *dev = adev_to_drm(adev);
1754         struct drm_file *file;
1755         int r;
1756
1757         r = mutex_lock_interruptible(&dev->filelist_mutex);
1758         if (r)
1759                 return r;
1760
1761         list_for_each_entry(file, &dev->filelist, lhead) {
1762                 struct amdgpu_fpriv *fpriv = file->driver_priv;
1763                 struct amdgpu_vm *vm = &fpriv->vm;
1764
1765                 seq_printf(m, "pid:%d\tProcess:%s ----------\n",
1766                                 vm->task_info.pid, vm->task_info.process_name);
1767                 r = amdgpu_bo_reserve(vm->root.bo, true);
1768                 if (r)
1769                         break;
1770                 amdgpu_debugfs_vm_bo_info(vm, m);
1771                 amdgpu_bo_unreserve(vm->root.bo);
1772         }
1773
1774         mutex_unlock(&dev->filelist_mutex);
1775
1776         return r;
1777 }
1778
1779 DEFINE_SHOW_ATTRIBUTE(amdgpu_debugfs_test_ib);
1780 DEFINE_SHOW_ATTRIBUTE(amdgpu_debugfs_vm_info);
1781 DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_evict_vram_fops, amdgpu_debugfs_evict_vram,
1782                          NULL, "%lld\n");
1783 DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_evict_gtt_fops, amdgpu_debugfs_evict_gtt,
1784                          NULL, "%lld\n");
1785 DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_benchmark_fops, NULL, amdgpu_debugfs_benchmark,
1786                          "%lld\n");
1787
1788 static void amdgpu_ib_preempt_fences_swap(struct amdgpu_ring *ring,
1789                                           struct dma_fence **fences)
1790 {
1791         struct amdgpu_fence_driver *drv = &ring->fence_drv;
1792         uint32_t sync_seq, last_seq;
1793
1794         last_seq = atomic_read(&ring->fence_drv.last_seq);
1795         sync_seq = ring->fence_drv.sync_seq;
1796
1797         last_seq &= drv->num_fences_mask;
1798         sync_seq &= drv->num_fences_mask;
1799
1800         do {
1801                 struct dma_fence *fence, **ptr;
1802
1803                 ++last_seq;
1804                 last_seq &= drv->num_fences_mask;
1805                 ptr = &drv->fences[last_seq];
1806
1807                 fence = rcu_dereference_protected(*ptr, 1);
1808                 RCU_INIT_POINTER(*ptr, NULL);
1809
1810                 if (!fence)
1811                         continue;
1812
1813                 fences[last_seq] = fence;
1814
1815         } while (last_seq != sync_seq);
1816 }
1817
1818 static void amdgpu_ib_preempt_signal_fences(struct dma_fence **fences,
1819                                             int length)
1820 {
1821         int i;
1822         struct dma_fence *fence;
1823
1824         for (i = 0; i < length; i++) {
1825                 fence = fences[i];
1826                 if (!fence)
1827                         continue;
1828                 dma_fence_signal(fence);
1829                 dma_fence_put(fence);
1830         }
1831 }
1832
1833 static void amdgpu_ib_preempt_job_recovery(struct drm_gpu_scheduler *sched)
1834 {
1835         struct drm_sched_job *s_job;
1836         struct dma_fence *fence;
1837
1838         spin_lock(&sched->job_list_lock);
1839         list_for_each_entry(s_job, &sched->pending_list, list) {
1840                 fence = sched->ops->run_job(s_job);
1841                 dma_fence_put(fence);
1842         }
1843         spin_unlock(&sched->job_list_lock);
1844 }
1845
1846 static void amdgpu_ib_preempt_mark_partial_job(struct amdgpu_ring *ring)
1847 {
1848         struct amdgpu_job *job;
1849         struct drm_sched_job *s_job, *tmp;
1850         uint32_t preempt_seq;
1851         struct dma_fence *fence, **ptr;
1852         struct amdgpu_fence_driver *drv = &ring->fence_drv;
1853         struct drm_gpu_scheduler *sched = &ring->sched;
1854         bool preempted = true;
1855
1856         if (ring->funcs->type != AMDGPU_RING_TYPE_GFX)
1857                 return;
1858
1859         preempt_seq = le32_to_cpu(*(drv->cpu_addr + 2));
1860         if (preempt_seq <= atomic_read(&drv->last_seq)) {
1861                 preempted = false;
1862                 goto no_preempt;
1863         }
1864
1865         preempt_seq &= drv->num_fences_mask;
1866         ptr = &drv->fences[preempt_seq];
1867         fence = rcu_dereference_protected(*ptr, 1);
1868
1869 no_preempt:
1870         spin_lock(&sched->job_list_lock);
1871         list_for_each_entry_safe(s_job, tmp, &sched->pending_list, list) {
1872                 if (dma_fence_is_signaled(&s_job->s_fence->finished)) {
1873                         /* remove job from ring_mirror_list */
1874                         list_del_init(&s_job->list);
1875                         sched->ops->free_job(s_job);
1876                         continue;
1877                 }
1878                 job = to_amdgpu_job(s_job);
1879                 if (preempted && (&job->hw_fence) == fence)
1880                         /* mark the job as preempted */
1881                         job->preemption_status |= AMDGPU_IB_PREEMPTED;
1882         }
1883         spin_unlock(&sched->job_list_lock);
1884 }
1885
1886 static int amdgpu_debugfs_ib_preempt(void *data, u64 val)
1887 {
1888         int r, length;
1889         struct amdgpu_ring *ring;
1890         struct dma_fence **fences = NULL;
1891         struct amdgpu_device *adev = (struct amdgpu_device *)data;
1892
1893         if (val >= AMDGPU_MAX_RINGS)
1894                 return -EINVAL;
1895
1896         ring = adev->rings[val];
1897
1898         if (!ring || !ring->funcs->preempt_ib || !ring->sched.thread)
1899                 return -EINVAL;
1900
1901         /* the last preemption failed */
1902         if (ring->trail_seq != le32_to_cpu(*ring->trail_fence_cpu_addr))
1903                 return -EBUSY;
1904
1905         length = ring->fence_drv.num_fences_mask + 1;
1906         fences = kcalloc(length, sizeof(void *), GFP_KERNEL);
1907         if (!fences)
1908                 return -ENOMEM;
1909
1910         /* Avoid accidently unparking the sched thread during GPU reset */
1911         r = down_read_killable(&adev->reset_domain->sem);
1912         if (r)
1913                 goto pro_end;
1914
1915         /* stop the scheduler */
1916         kthread_park(ring->sched.thread);
1917
1918         /* preempt the IB */
1919         r = amdgpu_ring_preempt_ib(ring);
1920         if (r) {
1921                 DRM_WARN("failed to preempt ring %d\n", ring->idx);
1922                 goto failure;
1923         }
1924
1925         amdgpu_fence_process(ring);
1926
1927         if (atomic_read(&ring->fence_drv.last_seq) !=
1928             ring->fence_drv.sync_seq) {
1929                 DRM_INFO("ring %d was preempted\n", ring->idx);
1930
1931                 amdgpu_ib_preempt_mark_partial_job(ring);
1932
1933                 /* swap out the old fences */
1934                 amdgpu_ib_preempt_fences_swap(ring, fences);
1935
1936                 amdgpu_fence_driver_force_completion(ring);
1937
1938                 /* resubmit unfinished jobs */
1939                 amdgpu_ib_preempt_job_recovery(&ring->sched);
1940
1941                 /* wait for jobs finished */
1942                 amdgpu_fence_wait_empty(ring);
1943
1944                 /* signal the old fences */
1945                 amdgpu_ib_preempt_signal_fences(fences, length);
1946         }
1947
1948 failure:
1949         /* restart the scheduler */
1950         kthread_unpark(ring->sched.thread);
1951
1952         up_read(&adev->reset_domain->sem);
1953
1954 pro_end:
1955         kfree(fences);
1956
1957         return r;
1958 }
1959
1960 static int amdgpu_debugfs_sclk_set(void *data, u64 val)
1961 {
1962         int ret = 0;
1963         uint32_t max_freq, min_freq;
1964         struct amdgpu_device *adev = (struct amdgpu_device *)data;
1965
1966         if (amdgpu_sriov_vf(adev) && !amdgpu_sriov_is_pp_one_vf(adev))
1967                 return -EINVAL;
1968
1969         ret = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1970         if (ret < 0) {
1971                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1972                 return ret;
1973         }
1974
1975         ret = amdgpu_dpm_get_dpm_freq_range(adev, PP_SCLK, &min_freq, &max_freq);
1976         if (ret == -EOPNOTSUPP) {
1977                 ret = 0;
1978                 goto out;
1979         }
1980         if (ret || val > max_freq || val < min_freq) {
1981                 ret = -EINVAL;
1982                 goto out;
1983         }
1984
1985         ret = amdgpu_dpm_set_soft_freq_range(adev, PP_SCLK, (uint32_t)val, (uint32_t)val);
1986         if (ret)
1987                 ret = -EINVAL;
1988
1989 out:
1990         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1991         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1992
1993         return ret;
1994 }
1995
1996 DEFINE_DEBUGFS_ATTRIBUTE(fops_ib_preempt, NULL,
1997                         amdgpu_debugfs_ib_preempt, "%llu\n");
1998
1999 DEFINE_DEBUGFS_ATTRIBUTE(fops_sclk_set, NULL,
2000                         amdgpu_debugfs_sclk_set, "%llu\n");
2001
2002 static ssize_t amdgpu_reset_dump_register_list_read(struct file *f,
2003                                 char __user *buf, size_t size, loff_t *pos)
2004 {
2005         struct amdgpu_device *adev = (struct amdgpu_device *)file_inode(f)->i_private;
2006         char reg_offset[12];
2007         int i, ret, len = 0;
2008
2009         if (*pos)
2010                 return 0;
2011
2012         memset(reg_offset, 0, 12);
2013         ret = down_read_killable(&adev->reset_domain->sem);
2014         if (ret)
2015                 return ret;
2016
2017         for (i = 0; i < adev->num_regs; i++) {
2018                 sprintf(reg_offset, "0x%x\n", adev->reset_dump_reg_list[i]);
2019                 up_read(&adev->reset_domain->sem);
2020                 if (copy_to_user(buf + len, reg_offset, strlen(reg_offset)))
2021                         return -EFAULT;
2022
2023                 len += strlen(reg_offset);
2024                 ret = down_read_killable(&adev->reset_domain->sem);
2025                 if (ret)
2026                         return ret;
2027         }
2028
2029         up_read(&adev->reset_domain->sem);
2030         *pos += len;
2031
2032         return len;
2033 }
2034
2035 static ssize_t amdgpu_reset_dump_register_list_write(struct file *f,
2036                         const char __user *buf, size_t size, loff_t *pos)
2037 {
2038         struct amdgpu_device *adev = (struct amdgpu_device *)file_inode(f)->i_private;
2039         char reg_offset[11];
2040         uint32_t *new = NULL, *tmp = NULL;
2041         int ret, i = 0, len = 0;
2042
2043         do {
2044                 memset(reg_offset, 0, 11);
2045                 if (copy_from_user(reg_offset, buf + len,
2046                                         min(10, ((int)size-len)))) {
2047                         ret = -EFAULT;
2048                         goto error_free;
2049                 }
2050
2051                 new = krealloc_array(tmp, i + 1, sizeof(uint32_t), GFP_KERNEL);
2052                 if (!new) {
2053                         ret = -ENOMEM;
2054                         goto error_free;
2055                 }
2056                 tmp = new;
2057                 if (sscanf(reg_offset, "%X %n", &tmp[i], &ret) != 1) {
2058                         ret = -EINVAL;
2059                         goto error_free;
2060                 }
2061
2062                 len += ret;
2063                 i++;
2064         } while (len < size);
2065
2066         new = kmalloc_array(i, sizeof(uint32_t), GFP_KERNEL);
2067         if (!new) {
2068                 ret = -ENOMEM;
2069                 goto error_free;
2070         }
2071         ret = down_write_killable(&adev->reset_domain->sem);
2072         if (ret)
2073                 goto error_free;
2074
2075         swap(adev->reset_dump_reg_list, tmp);
2076         swap(adev->reset_dump_reg_value, new);
2077         adev->num_regs = i;
2078         up_write(&adev->reset_domain->sem);
2079         ret = size;
2080
2081 error_free:
2082         if (tmp != new)
2083                 kfree(tmp);
2084         kfree(new);
2085         return ret;
2086 }
2087
2088 static const struct file_operations amdgpu_reset_dump_register_list = {
2089         .owner = THIS_MODULE,
2090         .read = amdgpu_reset_dump_register_list_read,
2091         .write = amdgpu_reset_dump_register_list_write,
2092         .llseek = default_llseek
2093 };
2094
2095 int amdgpu_debugfs_init(struct amdgpu_device *adev)
2096 {
2097         struct dentry *root = adev_to_drm(adev)->primary->debugfs_root;
2098         struct dentry *ent;
2099         int r, i;
2100
2101         if (!debugfs_initialized())
2102                 return 0;
2103
2104         debugfs_create_x32("amdgpu_smu_debug", 0600, root,
2105                            &adev->pm.smu_debug_mask);
2106
2107         ent = debugfs_create_file("amdgpu_preempt_ib", 0600, root, adev,
2108                                   &fops_ib_preempt);
2109         if (IS_ERR(ent)) {
2110                 DRM_ERROR("unable to create amdgpu_preempt_ib debugsfs file\n");
2111                 return PTR_ERR(ent);
2112         }
2113
2114         ent = debugfs_create_file("amdgpu_force_sclk", 0200, root, adev,
2115                                   &fops_sclk_set);
2116         if (IS_ERR(ent)) {
2117                 DRM_ERROR("unable to create amdgpu_set_sclk debugsfs file\n");
2118                 return PTR_ERR(ent);
2119         }
2120
2121         /* Register debugfs entries for amdgpu_ttm */
2122         amdgpu_ttm_debugfs_init(adev);
2123         amdgpu_debugfs_pm_init(adev);
2124         amdgpu_debugfs_sa_init(adev);
2125         amdgpu_debugfs_fence_init(adev);
2126         amdgpu_debugfs_gem_init(adev);
2127
2128         r = amdgpu_debugfs_regs_init(adev);
2129         if (r)
2130                 DRM_ERROR("registering register debugfs failed (%d).\n", r);
2131
2132         amdgpu_debugfs_firmware_init(adev);
2133         amdgpu_ta_if_debugfs_init(adev);
2134
2135 #if defined(CONFIG_DRM_AMD_DC)
2136         if (adev->dc_enabled)
2137                 dtn_debugfs_init(adev);
2138 #endif
2139
2140         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
2141                 struct amdgpu_ring *ring = adev->rings[i];
2142
2143                 if (!ring)
2144                         continue;
2145
2146                 amdgpu_debugfs_ring_init(adev, ring);
2147         }
2148
2149         for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
2150                 if (!amdgpu_vcnfw_log)
2151                         break;
2152
2153                 if (adev->vcn.harvest_config & (1 << i))
2154                         continue;
2155
2156                 amdgpu_debugfs_vcn_fwlog_init(adev, i, &adev->vcn.inst[i]);
2157         }
2158
2159         amdgpu_ras_debugfs_create_all(adev);
2160         amdgpu_rap_debugfs_init(adev);
2161         amdgpu_securedisplay_debugfs_init(adev);
2162         amdgpu_fw_attestation_debugfs_init(adev);
2163
2164         debugfs_create_file("amdgpu_evict_vram", 0444, root, adev,
2165                             &amdgpu_evict_vram_fops);
2166         debugfs_create_file("amdgpu_evict_gtt", 0444, root, adev,
2167                             &amdgpu_evict_gtt_fops);
2168         debugfs_create_file("amdgpu_test_ib", 0444, root, adev,
2169                             &amdgpu_debugfs_test_ib_fops);
2170         debugfs_create_file("amdgpu_vm_info", 0444, root, adev,
2171                             &amdgpu_debugfs_vm_info_fops);
2172         debugfs_create_file("amdgpu_benchmark", 0200, root, adev,
2173                             &amdgpu_benchmark_fops);
2174         debugfs_create_file("amdgpu_reset_dump_register_list", 0644, root, adev,
2175                             &amdgpu_reset_dump_register_list);
2176
2177         adev->debugfs_vbios_blob.data = adev->bios;
2178         adev->debugfs_vbios_blob.size = adev->bios_size;
2179         debugfs_create_blob("amdgpu_vbios", 0444, root,
2180                             &adev->debugfs_vbios_blob);
2181
2182         adev->debugfs_discovery_blob.data = adev->mman.discovery_bin;
2183         adev->debugfs_discovery_blob.size = adev->mman.discovery_tmr_size;
2184         debugfs_create_blob("amdgpu_discovery", 0444, root,
2185                             &adev->debugfs_discovery_blob);
2186
2187         return 0;
2188 }
2189
2190 #else
2191 int amdgpu_debugfs_init(struct amdgpu_device *adev)
2192 {
2193         return 0;
2194 }
2195 int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
2196 {
2197         return 0;
2198 }
2199 #endif