drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c

   1 /*
   2  * Copyright 2008 Advanced Micro Devices, Inc.
   3  * Copyright 2008 Red Hat Inc.
   4  * Copyright 2009 Jerome Glisse.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22  * OTHER DEALINGS IN THE SOFTWARE.
  23  *
  24  */
  25
  26 #include <linux/kthread.h>
  27 #include <linux/pci.h>
  28 #include <linux/uaccess.h>
  29 #include <linux/pm_runtime.h>
  30
  31 #include "amdgpu.h"
  32 #include "amdgpu_pm.h"
  33 #include "amdgpu_dm_debugfs.h"
  34 #include "amdgpu_ras.h"
  35 #include "amdgpu_rap.h"
  36 #include "amdgpu_securedisplay.h"
  37 #include "amdgpu_fw_attestation.h"
  38 #include "amdgpu_umr.h"
  39
  40 #include "amdgpu_reset.h"
  41 #include "amdgpu_psp_ta.h"
  42
  43 #if defined(CONFIG_DEBUG_FS)
  44
  45 /**
  46  * amdgpu_debugfs_process_reg_op - Handle MMIO register reads/writes
  47  *
  48  * @read: True if reading
  49  * @f: open file handle
  50  * @buf: User buffer to write/read to
  51  * @size: Number of bytes to write/read
  52  * @pos:  Offset to seek to
  53  *
  54  * This debugfs entry has special meaning on the offset being sought.
  55  * Various bits have different meanings:
  56  *
  57  * Bit 62:  Indicates a GRBM bank switch is needed
  58  * Bit 61:  Indicates a SRBM bank switch is needed (implies bit 62 is
  59  *          zero)
  60  * Bits 24..33: The SE or ME selector if needed
  61  * Bits 34..43: The SH (or SA) or PIPE selector if needed
  62  * Bits 44..53: The INSTANCE (or CU/WGP) or QUEUE selector if needed
  63  *
  64  * Bit 23:  Indicates that the PM power gating lock should be held
  65  *          This is necessary to read registers that might be
  66  *          unreliable during a power gating transistion.
  67  *
  68  * The lower bits are the BYTE offset of the register to read.  This
  69  * allows reading multiple registers in a single call and having
  70  * the returned size reflect that.
  71  */
  72 static int  amdgpu_debugfs_process_reg_op(bool read, struct file *f,
  73                 char __user *buf, size_t size, loff_t *pos)
  74 {
  75         struct amdgpu_device *adev = file_inode(f)->i_private;
  76         ssize_t result = 0;
  77         int r;
  78         bool pm_pg_lock, use_bank, use_ring;
  79         unsigned int instance_bank, sh_bank, se_bank, me, pipe, queue, vmid;
  80
  81         pm_pg_lock = use_bank = use_ring = false;
  82         instance_bank = sh_bank = se_bank = me = pipe = queue = vmid = 0;
  83
  84         if (size & 0x3 || *pos & 0x3 ||
  85                         ((*pos & (1ULL << 62)) && (*pos & (1ULL << 61))))
  86                 return -EINVAL;
  87
  88         /* are we reading registers for which a PG lock is necessary? */
  89         pm_pg_lock = (*pos >> 23) & 1;
  90
  91         if (*pos & (1ULL << 62)) {
  92                 se_bank = (*pos & GENMASK_ULL(33, 24)) >> 24;
  93                 sh_bank = (*pos & GENMASK_ULL(43, 34)) >> 34;
  94                 instance_bank = (*pos & GENMASK_ULL(53, 44)) >> 44;
  95
  96                 if (se_bank == 0x3FF)
  97                         se_bank = 0xFFFFFFFF;
  98                 if (sh_bank == 0x3FF)
  99                         sh_bank = 0xFFFFFFFF;
 100                 if (instance_bank == 0x3FF)
 101                         instance_bank = 0xFFFFFFFF;
 102                 use_bank = true;
 103         } else if (*pos & (1ULL << 61)) {
 104
 105                 me = (*pos & GENMASK_ULL(33, 24)) >> 24;
 106                 pipe = (*pos & GENMASK_ULL(43, 34)) >> 34;
 107                 queue = (*pos & GENMASK_ULL(53, 44)) >> 44;
 108                 vmid = (*pos & GENMASK_ULL(58, 54)) >> 54;
 109
 110                 use_ring = true;
 111         } else {
 112                 use_bank = use_ring = false;
 113         }
 114
 115         *pos &= (1UL << 22) - 1;
 116
 117         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 118         if (r < 0) {
 119                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 120                 return r;
 121         }
 122
 123         r = amdgpu_virt_enable_access_debugfs(adev);
 124         if (r < 0) {
 125                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 126                 return r;
 127         }
 128
 129         if (use_bank) {
 130                 if ((sh_bank != 0xFFFFFFFF && sh_bank >= adev->gfx.config.max_sh_per_se) ||
 131                     (se_bank != 0xFFFFFFFF && se_bank >= adev->gfx.config.max_shader_engines)) {
 132                         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 133                         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 134                         amdgpu_virt_disable_access_debugfs(adev);
 135                         return -EINVAL;
 136                 }
 137                 mutex_lock(&adev->grbm_idx_mutex);
 138                 amdgpu_gfx_select_se_sh(adev, se_bank,
 139                                         sh_bank, instance_bank, 0);
 140         } else if (use_ring) {
 141                 mutex_lock(&adev->srbm_mutex);
 142                 amdgpu_gfx_select_me_pipe_q(adev, me, pipe, queue, vmid, 0);
 143         }
 144
 145         if (pm_pg_lock)
 146                 mutex_lock(&adev->pm.mutex);
 147
 148         while (size) {
 149                 uint32_t value;
 150
 151                 if (read) {
 152                         value = RREG32(*pos >> 2);
 153                         r = put_user(value, (uint32_t *)buf);
 154                 } else {
 155                         r = get_user(value, (uint32_t *)buf);
 156                         if (!r)
 157                                 amdgpu_mm_wreg_mmio_rlc(adev, *pos >> 2, value, 0);
 158                 }
 159                 if (r) {
 160                         result = r;
 161                         goto end;
 162                 }
 163
 164                 result += 4;
 165                 buf += 4;
 166                 *pos += 4;
 167                 size -= 4;
 168         }
 169
 170 end:
 171         if (use_bank) {
 172                 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
 173                 mutex_unlock(&adev->grbm_idx_mutex);
 174         } else if (use_ring) {
 175                 amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0);
 176                 mutex_unlock(&adev->srbm_mutex);
 177         }
 178
 179         if (pm_pg_lock)
 180                 mutex_unlock(&adev->pm.mutex);
 181
 182         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 183         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 184
 185         amdgpu_virt_disable_access_debugfs(adev);
 186         return result;
 187 }
 188
 189 /*
 190  * amdgpu_debugfs_regs_read - Callback for reading MMIO registers
 191  */
 192 static ssize_t amdgpu_debugfs_regs_read(struct file *f, char __user *buf,
 193                                         size_t size, loff_t *pos)
 194 {
 195         return amdgpu_debugfs_process_reg_op(true, f, buf, size, pos);
 196 }
 197
 198 /*
 199  * amdgpu_debugfs_regs_write - Callback for writing MMIO registers
 200  */
 201 static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,
 202                                          size_t size, loff_t *pos)
 203 {
 204         return amdgpu_debugfs_process_reg_op(false, f, (char __user *)buf, size, pos);
 205 }
 206
 207 static int amdgpu_debugfs_regs2_open(struct inode *inode, struct file *file)
 208 {
 209         struct amdgpu_debugfs_regs2_data *rd;
 210
 211         rd = kzalloc(sizeof(*rd), GFP_KERNEL);
 212         if (!rd)
 213                 return -ENOMEM;
 214         rd->adev = file_inode(file)->i_private;
 215         file->private_data = rd;
 216         mutex_init(&rd->lock);
 217
 218         return 0;
 219 }
 220
 221 static int amdgpu_debugfs_regs2_release(struct inode *inode, struct file *file)
 222 {
 223         struct amdgpu_debugfs_regs2_data *rd = file->private_data;
 224
 225         mutex_destroy(&rd->lock);
 226         kfree(file->private_data);
 227         return 0;
 228 }
 229
 230 static ssize_t amdgpu_debugfs_regs2_op(struct file *f, char __user *buf, u32 offset, size_t size, int write_en)
 231 {
 232         struct amdgpu_debugfs_regs2_data *rd = f->private_data;
 233         struct amdgpu_device *adev = rd->adev;
 234         ssize_t result = 0;
 235         int r;
 236         uint32_t value;
 237
 238         if (size & 0x3 || offset & 0x3)
 239                 return -EINVAL;
 240
 241         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 242         if (r < 0) {
 243                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 244                 return r;
 245         }
 246
 247         r = amdgpu_virt_enable_access_debugfs(adev);
 248         if (r < 0) {
 249                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 250                 return r;
 251         }
 252
 253         mutex_lock(&rd->lock);
 254
 255         if (rd->id.use_grbm) {
 256                 if ((rd->id.grbm.sh != 0xFFFFFFFF && rd->id.grbm.sh >= adev->gfx.config.max_sh_per_se) ||
 257                     (rd->id.grbm.se != 0xFFFFFFFF && rd->id.grbm.se >= adev->gfx.config.max_shader_engines)) {
 258                         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 259                         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 260                         amdgpu_virt_disable_access_debugfs(adev);
 261                         mutex_unlock(&rd->lock);
 262                         return -EINVAL;
 263                 }
 264                 mutex_lock(&adev->grbm_idx_mutex);
 265                 amdgpu_gfx_select_se_sh(adev, rd->id.grbm.se,
 266                                                   rd->id.grbm.sh,
 267                                                   rd->id.grbm.instance, rd->id.xcc_id);
 268         }
 269
 270         if (rd->id.use_srbm) {
 271                 mutex_lock(&adev->srbm_mutex);
 272                 amdgpu_gfx_select_me_pipe_q(adev, rd->id.srbm.me, rd->id.srbm.pipe,
 273                                             rd->id.srbm.queue, rd->id.srbm.vmid, rd->id.xcc_id);
 274         }
 275
 276         if (rd->id.pg_lock)
 277                 mutex_lock(&adev->pm.mutex);
 278
 279         while (size) {
 280                 if (!write_en) {
 281                         value = RREG32(offset >> 2);
 282                         r = put_user(value, (uint32_t *)buf);
 283                 } else {
 284                         r = get_user(value, (uint32_t *)buf);
 285                         if (!r)
 286                                 amdgpu_mm_wreg_mmio_rlc(adev, offset >> 2, value, rd->id.xcc_id);
 287                 }
 288                 if (r) {
 289                         result = r;
 290                         goto end;
 291                 }
 292                 offset += 4;
 293                 size -= 4;
 294                 result += 4;
 295                 buf += 4;
 296         }
 297 end:
 298         if (rd->id.use_grbm) {
 299                 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, rd->id.xcc_id);
 300                 mutex_unlock(&adev->grbm_idx_mutex);
 301         }
 302
 303         if (rd->id.use_srbm) {
 304                 amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, rd->id.xcc_id);
 305                 mutex_unlock(&adev->srbm_mutex);
 306         }
 307
 308         if (rd->id.pg_lock)
 309                 mutex_unlock(&adev->pm.mutex);
 310
 311         mutex_unlock(&rd->lock);
 312
 313         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 314         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 315
 316         amdgpu_virt_disable_access_debugfs(adev);
 317         return result;
 318 }
 319
 320 static long amdgpu_debugfs_regs2_ioctl(struct file *f, unsigned int cmd, unsigned long data)
 321 {
 322         struct amdgpu_debugfs_regs2_data *rd = f->private_data;
 323         struct amdgpu_debugfs_regs2_iocdata v1_data;
 324         int r;
 325
 326         mutex_lock(&rd->lock);
 327
 328         switch (cmd) {
 329         case AMDGPU_DEBUGFS_REGS2_IOC_SET_STATE_V2:
 330                 r = copy_from_user(&rd->id, (struct amdgpu_debugfs_regs2_iocdata_v2 *)data,
 331                                    sizeof(rd->id));
 332                 if (r)
 333                         r = -EINVAL;
 334                 goto done;
 335         case AMDGPU_DEBUGFS_REGS2_IOC_SET_STATE:
 336                 r = copy_from_user(&v1_data, (struct amdgpu_debugfs_regs2_iocdata *)data,
 337                                    sizeof(v1_data));
 338                 if (r) {
 339                         r = -EINVAL;
 340                         goto done;
 341                 }
 342                 goto v1_copy;
 343         default:
 344                 r = -EINVAL;
 345                 goto done;
 346         }
 347
 348 v1_copy:
 349         rd->id.use_srbm = v1_data.use_srbm;
 350         rd->id.use_grbm = v1_data.use_grbm;
 351         rd->id.pg_lock = v1_data.pg_lock;
 352         rd->id.grbm.se = v1_data.grbm.se;
 353         rd->id.grbm.sh = v1_data.grbm.sh;
 354         rd->id.grbm.instance = v1_data.grbm.instance;
 355         rd->id.srbm.me = v1_data.srbm.me;
 356         rd->id.srbm.pipe = v1_data.srbm.pipe;
 357         rd->id.srbm.queue = v1_data.srbm.queue;
 358         rd->id.xcc_id = 0;
 359 done:
 360         mutex_unlock(&rd->lock);
 361         return r;
 362 }
 363
 364 static ssize_t amdgpu_debugfs_regs2_read(struct file *f, char __user *buf, size_t size, loff_t *pos)
 365 {
 366         return amdgpu_debugfs_regs2_op(f, buf, *pos, size, 0);
 367 }
 368
 369 static ssize_t amdgpu_debugfs_regs2_write(struct file *f, const char __user *buf, size_t size, loff_t *pos)
 370 {
 371         return amdgpu_debugfs_regs2_op(f, (char __user *)buf, *pos, size, 1);
 372 }
 373
 374 static int amdgpu_debugfs_gprwave_open(struct inode *inode, struct file *file)
 375 {
 376         struct amdgpu_debugfs_gprwave_data *rd;
 377
 378         rd = kzalloc(sizeof(*rd), GFP_KERNEL);
 379         if (!rd)
 380                 return -ENOMEM;
 381         rd->adev = file_inode(file)->i_private;
 382         file->private_data = rd;
 383         mutex_init(&rd->lock);
 384
 385         return 0;
 386 }
 387
 388 static int amdgpu_debugfs_gprwave_release(struct inode *inode, struct file *file)
 389 {
 390         struct amdgpu_debugfs_gprwave_data *rd = file->private_data;
 391
 392         mutex_destroy(&rd->lock);
 393         kfree(file->private_data);
 394         return 0;
 395 }
 396
 397 static ssize_t amdgpu_debugfs_gprwave_read(struct file *f, char __user *buf, size_t size, loff_t *pos)
 398 {
 399         struct amdgpu_debugfs_gprwave_data *rd = f->private_data;
 400         struct amdgpu_device *adev = rd->adev;
 401         ssize_t result = 0;
 402         int r;
 403         uint32_t *data, x;
 404
 405         if (size & 0x3 || *pos & 0x3)
 406                 return -EINVAL;
 407
 408         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 409         if (r < 0) {
 410                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 411                 return r;
 412         }
 413
 414         r = amdgpu_virt_enable_access_debugfs(adev);
 415         if (r < 0) {
 416                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 417                 return r;
 418         }
 419
 420         data = kcalloc(1024, sizeof(*data), GFP_KERNEL);
 421         if (!data) {
 422                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 423                 amdgpu_virt_disable_access_debugfs(adev);
 424                 return -ENOMEM;
 425         }
 426
 427         /* switch to the specific se/sh/cu */
 428         mutex_lock(&adev->grbm_idx_mutex);
 429         amdgpu_gfx_select_se_sh(adev, rd->id.se, rd->id.sh, rd->id.cu, rd->id.xcc_id);
 430
 431         if (!rd->id.gpr_or_wave) {
 432                 x = 0;
 433                 if (adev->gfx.funcs->read_wave_data)
 434                         adev->gfx.funcs->read_wave_data(adev, rd->id.xcc_id, rd->id.simd, rd->id.wave, data, &x);
 435         } else {
 436                 x = size >> 2;
 437                 if (rd->id.gpr.vpgr_or_sgpr) {
 438                         if (adev->gfx.funcs->read_wave_vgprs)
 439                                 adev->gfx.funcs->read_wave_vgprs(adev, rd->id.xcc_id, rd->id.simd, rd->id.wave, rd->id.gpr.thread, *pos, size>>2, data);
 440                 } else {
 441                         if (adev->gfx.funcs->read_wave_sgprs)
 442                                 adev->gfx.funcs->read_wave_sgprs(adev, rd->id.xcc_id, rd->id.simd, rd->id.wave, *pos, size>>2, data);
 443                 }
 444         }
 445
 446         amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, rd->id.xcc_id);
 447         mutex_unlock(&adev->grbm_idx_mutex);
 448
 449         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 450         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 451
 452         if (!x) {
 453                 result = -EINVAL;
 454                 goto done;
 455         }
 456
 457         while (size && (*pos < x * 4)) {
 458                 uint32_t value;
 459
 460                 value = data[*pos >> 2];
 461                 r = put_user(value, (uint32_t *)buf);
 462                 if (r) {
 463                         result = r;
 464                         goto done;
 465                 }
 466
 467                 result += 4;
 468                 buf += 4;
 469                 *pos += 4;
 470                 size -= 4;
 471         }
 472
 473 done:
 474         amdgpu_virt_disable_access_debugfs(adev);
 475         kfree(data);
 476         return result;
 477 }
 478
 479 static long amdgpu_debugfs_gprwave_ioctl(struct file *f, unsigned int cmd, unsigned long data)
 480 {
 481         struct amdgpu_debugfs_gprwave_data *rd = f->private_data;
 482         int r = 0;
 483
 484         mutex_lock(&rd->lock);
 485
 486         switch (cmd) {
 487         case AMDGPU_DEBUGFS_GPRWAVE_IOC_SET_STATE:
 488                 if (copy_from_user(&rd->id,
 489                                    (struct amdgpu_debugfs_gprwave_iocdata *)data,
 490                                    sizeof(rd->id)))
 491                         r = -EFAULT;
 492                 goto done;
 493         default:
 494                 r = -EINVAL;
 495                 goto done;
 496         }
 497
 498 done:
 499         mutex_unlock(&rd->lock);
 500         return r;
 501 }
 502
 503
 504
 505
 506 /**
 507  * amdgpu_debugfs_regs_pcie_read - Read from a PCIE register
 508  *
 509  * @f: open file handle
 510  * @buf: User buffer to store read data in
 511  * @size: Number of bytes to read
 512  * @pos:  Offset to seek to
 513  *
 514  * The lower bits are the BYTE offset of the register to read.  This
 515  * allows reading multiple registers in a single call and having
 516  * the returned size reflect that.
 517  */
 518 static ssize_t amdgpu_debugfs_regs_pcie_read(struct file *f, char __user *buf,
 519                                         size_t size, loff_t *pos)
 520 {
 521         struct amdgpu_device *adev = file_inode(f)->i_private;
 522         ssize_t result = 0;
 523         int r;
 524
 525         if (size & 0x3 || *pos & 0x3)
 526                 return -EINVAL;
 527
 528         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 529         if (r < 0) {
 530                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 531                 return r;
 532         }
 533
 534         r = amdgpu_virt_enable_access_debugfs(adev);
 535         if (r < 0) {
 536                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 537                 return r;
 538         }
 539
 540         while (size) {
 541                 uint32_t value;
 542
 543                 value = RREG32_PCIE(*pos);
 544                 r = put_user(value, (uint32_t *)buf);
 545                 if (r)
 546                         goto out;
 547
 548                 result += 4;
 549                 buf += 4;
 550                 *pos += 4;
 551                 size -= 4;
 552         }
 553
 554         r = result;
 555 out:
 556         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 557         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 558         amdgpu_virt_disable_access_debugfs(adev);
 559         return r;
 560 }
 561
 562 /**
 563  * amdgpu_debugfs_regs_pcie_write - Write to a PCIE register
 564  *
 565  * @f: open file handle
 566  * @buf: User buffer to write data from
 567  * @size: Number of bytes to write
 568  * @pos:  Offset to seek to
 569  *
 570  * The lower bits are the BYTE offset of the register to write.  This
 571  * allows writing multiple registers in a single call and having
 572  * the returned size reflect that.
 573  */
 574 static ssize_t amdgpu_debugfs_regs_pcie_write(struct file *f, const char __user *buf,
 575                                          size_t size, loff_t *pos)
 576 {
 577         struct amdgpu_device *adev = file_inode(f)->i_private;
 578         ssize_t result = 0;
 579         int r;
 580
 581         if (size & 0x3 || *pos & 0x3)
 582                 return -EINVAL;
 583
 584         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 585         if (r < 0) {
 586                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 587                 return r;
 588         }
 589
 590         r = amdgpu_virt_enable_access_debugfs(adev);
 591         if (r < 0) {
 592                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 593                 return r;
 594         }
 595
 596         while (size) {
 597                 uint32_t value;
 598
 599                 r = get_user(value, (uint32_t *)buf);
 600                 if (r)
 601                         goto out;
 602
 603                 WREG32_PCIE(*pos, value);
 604
 605                 result += 4;
 606                 buf += 4;
 607                 *pos += 4;
 608                 size -= 4;
 609         }
 610
 611         r = result;
 612 out:
 613         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 614         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 615         amdgpu_virt_disable_access_debugfs(adev);
 616         return r;
 617 }
 618
 619 /**
 620  * amdgpu_debugfs_regs_didt_read - Read from a DIDT register
 621  *
 622  * @f: open file handle
 623  * @buf: User buffer to store read data in
 624  * @size: Number of bytes to read
 625  * @pos:  Offset to seek to
 626  *
 627  * The lower bits are the BYTE offset of the register to read.  This
 628  * allows reading multiple registers in a single call and having
 629  * the returned size reflect that.
 630  */
 631 static ssize_t amdgpu_debugfs_regs_didt_read(struct file *f, char __user *buf,
 632                                         size_t size, loff_t *pos)
 633 {
 634         struct amdgpu_device *adev = file_inode(f)->i_private;
 635         ssize_t result = 0;
 636         int r;
 637
 638         if (size & 0x3 || *pos & 0x3)
 639                 return -EINVAL;
 640
 641         if (!adev->didt_rreg)
 642                 return -EOPNOTSUPP;
 643
 644         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 645         if (r < 0) {
 646                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 647                 return r;
 648         }
 649
 650         r = amdgpu_virt_enable_access_debugfs(adev);
 651         if (r < 0) {
 652                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 653                 return r;
 654         }
 655
 656         while (size) {
 657                 uint32_t value;
 658
 659                 value = RREG32_DIDT(*pos >> 2);
 660                 r = put_user(value, (uint32_t *)buf);
 661                 if (r)
 662                         goto out;
 663
 664                 result += 4;
 665                 buf += 4;
 666                 *pos += 4;
 667                 size -= 4;
 668         }
 669
 670         r = result;
 671 out:
 672         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 673         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 674         amdgpu_virt_disable_access_debugfs(adev);
 675         return r;
 676 }
 677
 678 /**
 679  * amdgpu_debugfs_regs_didt_write - Write to a DIDT register
 680  *
 681  * @f: open file handle
 682  * @buf: User buffer to write data from
 683  * @size: Number of bytes to write
 684  * @pos:  Offset to seek to
 685  *
 686  * The lower bits are the BYTE offset of the register to write.  This
 687  * allows writing multiple registers in a single call and having
 688  * the returned size reflect that.
 689  */
 690 static ssize_t amdgpu_debugfs_regs_didt_write(struct file *f, const char __user *buf,
 691                                          size_t size, loff_t *pos)
 692 {
 693         struct amdgpu_device *adev = file_inode(f)->i_private;
 694         ssize_t result = 0;
 695         int r;
 696
 697         if (size & 0x3 || *pos & 0x3)
 698                 return -EINVAL;
 699
 700         if (!adev->didt_wreg)
 701                 return -EOPNOTSUPP;
 702
 703         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 704         if (r < 0) {
 705                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 706                 return r;
 707         }
 708
 709         r = amdgpu_virt_enable_access_debugfs(adev);
 710         if (r < 0) {
 711                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 712                 return r;
 713         }
 714
 715         while (size) {
 716                 uint32_t value;
 717
 718                 r = get_user(value, (uint32_t *)buf);
 719                 if (r)
 720                         goto out;
 721
 722                 WREG32_DIDT(*pos >> 2, value);
 723
 724                 result += 4;
 725                 buf += 4;
 726                 *pos += 4;
 727                 size -= 4;
 728         }
 729
 730         r = result;
 731 out:
 732         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 733         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 734         amdgpu_virt_disable_access_debugfs(adev);
 735         return r;
 736 }
 737
 738 /**
 739  * amdgpu_debugfs_regs_smc_read - Read from a SMC register
 740  *
 741  * @f: open file handle
 742  * @buf: User buffer to store read data in
 743  * @size: Number of bytes to read
 744  * @pos:  Offset to seek to
 745  *
 746  * The lower bits are the BYTE offset of the register to read.  This
 747  * allows reading multiple registers in a single call and having
 748  * the returned size reflect that.
 749  */
 750 static ssize_t amdgpu_debugfs_regs_smc_read(struct file *f, char __user *buf,
 751                                         size_t size, loff_t *pos)
 752 {
 753         struct amdgpu_device *adev = file_inode(f)->i_private;
 754         ssize_t result = 0;
 755         int r;
 756
 757         if (!adev->smc_rreg)
 758                 return -EPERM;
 759
 760         if (size & 0x3 || *pos & 0x3)
 761                 return -EINVAL;
 762
 763         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 764         if (r < 0) {
 765                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 766                 return r;
 767         }
 768
 769         r = amdgpu_virt_enable_access_debugfs(adev);
 770         if (r < 0) {
 771                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 772                 return r;
 773         }
 774
 775         while (size) {
 776                 uint32_t value;
 777
 778                 value = RREG32_SMC(*pos);
 779                 r = put_user(value, (uint32_t *)buf);
 780                 if (r)
 781                         goto out;
 782
 783                 result += 4;
 784                 buf += 4;
 785                 *pos += 4;
 786                 size -= 4;
 787         }
 788
 789         r = result;
 790 out:
 791         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 792         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 793         amdgpu_virt_disable_access_debugfs(adev);
 794         return r;
 795 }
 796
 797 /**
 798  * amdgpu_debugfs_regs_smc_write - Write to a SMC register
 799  *
 800  * @f: open file handle
 801  * @buf: User buffer to write data from
 802  * @size: Number of bytes to write
 803  * @pos:  Offset to seek to
 804  *
 805  * The lower bits are the BYTE offset of the register to write.  This
 806  * allows writing multiple registers in a single call and having
 807  * the returned size reflect that.
 808  */
 809 static ssize_t amdgpu_debugfs_regs_smc_write(struct file *f, const char __user *buf,
 810                                          size_t size, loff_t *pos)
 811 {
 812         struct amdgpu_device *adev = file_inode(f)->i_private;
 813         ssize_t result = 0;
 814         int r;
 815
 816         if (!adev->smc_wreg)
 817                 return -EPERM;
 818
 819         if (size & 0x3 || *pos & 0x3)
 820                 return -EINVAL;
 821
 822         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 823         if (r < 0) {
 824                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 825                 return r;
 826         }
 827
 828         r = amdgpu_virt_enable_access_debugfs(adev);
 829         if (r < 0) {
 830                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 831                 return r;
 832         }
 833
 834         while (size) {
 835                 uint32_t value;
 836
 837                 r = get_user(value, (uint32_t *)buf);
 838                 if (r)
 839                         goto out;
 840
 841                 WREG32_SMC(*pos, value);
 842
 843                 result += 4;
 844                 buf += 4;
 845                 *pos += 4;
 846                 size -= 4;
 847         }
 848
 849         r = result;
 850 out:
 851         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 852         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 853         amdgpu_virt_disable_access_debugfs(adev);
 854         return r;
 855 }
 856
 857 /**
 858  * amdgpu_debugfs_gca_config_read - Read from gfx config data
 859  *
 860  * @f: open file handle
 861  * @buf: User buffer to store read data in
 862  * @size: Number of bytes to read
 863  * @pos:  Offset to seek to
 864  *
 865  * This file is used to access configuration data in a somewhat
 866  * stable fashion.  The format is a series of DWORDs with the first
 867  * indicating which revision it is.  New content is appended to the
 868  * end so that older software can still read the data.
 869  */
 870
 871 static ssize_t amdgpu_debugfs_gca_config_read(struct file *f, char __user *buf,
 872                                         size_t size, loff_t *pos)
 873 {
 874         struct amdgpu_device *adev = file_inode(f)->i_private;
 875         ssize_t result = 0;
 876         int r;
 877         uint32_t *config, no_regs = 0;
 878
 879         if (size & 0x3 || *pos & 0x3)
 880                 return -EINVAL;
 881
 882         config = kmalloc_array(256, sizeof(*config), GFP_KERNEL);
 883         if (!config)
 884                 return -ENOMEM;
 885
 886         /* version, increment each time something is added */
 887         config[no_regs++] = 5;
 888         config[no_regs++] = adev->gfx.config.max_shader_engines;
 889         config[no_regs++] = adev->gfx.config.max_tile_pipes;
 890         config[no_regs++] = adev->gfx.config.max_cu_per_sh;
 891         config[no_regs++] = adev->gfx.config.max_sh_per_se;
 892         config[no_regs++] = adev->gfx.config.max_backends_per_se;
 893         config[no_regs++] = adev->gfx.config.max_texture_channel_caches;
 894         config[no_regs++] = adev->gfx.config.max_gprs;
 895         config[no_regs++] = adev->gfx.config.max_gs_threads;
 896         config[no_regs++] = adev->gfx.config.max_hw_contexts;
 897         config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_frontend;
 898         config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_backend;
 899         config[no_regs++] = adev->gfx.config.sc_hiz_tile_fifo_size;
 900         config[no_regs++] = adev->gfx.config.sc_earlyz_tile_fifo_size;
 901         config[no_regs++] = adev->gfx.config.num_tile_pipes;
 902         config[no_regs++] = adev->gfx.config.backend_enable_mask;
 903         config[no_regs++] = adev->gfx.config.mem_max_burst_length_bytes;
 904         config[no_regs++] = adev->gfx.config.mem_row_size_in_kb;
 905         config[no_regs++] = adev->gfx.config.shader_engine_tile_size;
 906         config[no_regs++] = adev->gfx.config.num_gpus;
 907         config[no_regs++] = adev->gfx.config.multi_gpu_tile_size;
 908         config[no_regs++] = adev->gfx.config.mc_arb_ramcfg;
 909         config[no_regs++] = adev->gfx.config.gb_addr_config;
 910         config[no_regs++] = adev->gfx.config.num_rbs;
 911
 912         /* rev==1 */
 913         config[no_regs++] = adev->rev_id;
 914         config[no_regs++] = lower_32_bits(adev->pg_flags);
 915         config[no_regs++] = lower_32_bits(adev->cg_flags);
 916
 917         /* rev==2 */
 918         config[no_regs++] = adev->family;
 919         config[no_regs++] = adev->external_rev_id;
 920
 921         /* rev==3 */
 922         config[no_regs++] = adev->pdev->device;
 923         config[no_regs++] = adev->pdev->revision;
 924         config[no_regs++] = adev->pdev->subsystem_device;
 925         config[no_regs++] = adev->pdev->subsystem_vendor;
 926
 927         /* rev==4 APU flag */
 928         config[no_regs++] = adev->flags & AMD_IS_APU ? 1 : 0;
 929
 930         /* rev==5 PG/CG flag upper 32bit */
 931         config[no_regs++] = upper_32_bits(adev->pg_flags);
 932         config[no_regs++] = upper_32_bits(adev->cg_flags);
 933
 934         while (size && (*pos < no_regs * 4)) {
 935                 uint32_t value;
 936
 937                 value = config[*pos >> 2];
 938                 r = put_user(value, (uint32_t *)buf);
 939                 if (r) {
 940                         kfree(config);
 941                         return r;
 942                 }
 943
 944                 result += 4;
 945                 buf += 4;
 946                 *pos += 4;
 947                 size -= 4;
 948         }
 949
 950         kfree(config);
 951         return result;
 952 }
 953
 954 /**
 955  * amdgpu_debugfs_sensor_read - Read from the powerplay sensors
 956  *
 957  * @f: open file handle
 958  * @buf: User buffer to store read data in
 959  * @size: Number of bytes to read
 960  * @pos:  Offset to seek to
 961  *
 962  * The offset is treated as the BYTE address of one of the sensors
 963  * enumerated in amd/include/kgd_pp_interface.h under the
 964  * 'amd_pp_sensors' enumeration.  For instance to read the UVD VCLK
 965  * you would use the offset 3 * 4 = 12.
 966  */
 967 static ssize_t amdgpu_debugfs_sensor_read(struct file *f, char __user *buf,
 968                                         size_t size, loff_t *pos)
 969 {
 970         struct amdgpu_device *adev = file_inode(f)->i_private;
 971         int idx, x, outsize, r, valuesize;
 972         uint32_t values[16];
 973
 974         if (size & 3 || *pos & 0x3)
 975                 return -EINVAL;
 976
 977         if (!adev->pm.dpm_enabled)
 978                 return -EINVAL;
 979
 980         /* convert offset to sensor number */
 981         idx = *pos >> 2;
 982
 983         valuesize = sizeof(values);
 984
 985         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 986         if (r < 0) {
 987                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 988                 return r;
 989         }
 990
 991         r = amdgpu_virt_enable_access_debugfs(adev);
 992         if (r < 0) {
 993                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 994                 return r;
 995         }
 996
 997         r = amdgpu_dpm_read_sensor(adev, idx, &values[0], &valuesize);
 998
 999         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1000         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1001
1002         if (r) {
1003                 amdgpu_virt_disable_access_debugfs(adev);
1004                 return r;
1005         }
1006
1007         if (size > valuesize) {
1008                 amdgpu_virt_disable_access_debugfs(adev);
1009                 return -EINVAL;
1010         }
1011
1012         outsize = 0;
1013         x = 0;
1014         if (!r) {
1015                 while (size) {
1016                         r = put_user(values[x++], (int32_t *)buf);
1017                         buf += 4;
1018                         size -= 4;
1019                         outsize += 4;
1020                 }
1021         }
1022
1023         amdgpu_virt_disable_access_debugfs(adev);
1024         return !r ? outsize : r;
1025 }
1026
1027 /** amdgpu_debugfs_wave_read - Read WAVE STATUS data
1028  *
1029  * @f: open file handle
1030  * @buf: User buffer to store read data in
1031  * @size: Number of bytes to read
1032  * @pos:  Offset to seek to
1033  *
1034  * The offset being sought changes which wave that the status data
1035  * will be returned for.  The bits are used as follows:
1036  *
1037  * Bits 0..6:   Byte offset into data
1038  * Bits 7..14:  SE selector
1039  * Bits 15..22: SH/SA selector
1040  * Bits 23..30: CU/{WGP+SIMD} selector
1041  * Bits 31..36: WAVE ID selector
1042  * Bits 37..44: SIMD ID selector
1043  *
1044  * The returned data begins with one DWORD of version information
1045  * Followed by WAVE STATUS registers relevant to the GFX IP version
1046  * being used.  See gfx_v8_0_read_wave_data() for an example output.
1047  */
1048 static ssize_t amdgpu_debugfs_wave_read(struct file *f, char __user *buf,
1049                                         size_t size, loff_t *pos)
1050 {
1051         struct amdgpu_device *adev = f->f_inode->i_private;
1052         int r, x;
1053         ssize_t result = 0;
1054         uint32_t offset, se, sh, cu, wave, simd, data[32];
1055
1056         if (size & 3 || *pos & 3)
1057                 return -EINVAL;
1058
1059         /* decode offset */
1060         offset = (*pos & GENMASK_ULL(6, 0));
1061         se = (*pos & GENMASK_ULL(14, 7)) >> 7;
1062         sh = (*pos & GENMASK_ULL(22, 15)) >> 15;
1063         cu = (*pos & GENMASK_ULL(30, 23)) >> 23;
1064         wave = (*pos & GENMASK_ULL(36, 31)) >> 31;
1065         simd = (*pos & GENMASK_ULL(44, 37)) >> 37;
1066
1067         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1068         if (r < 0) {
1069                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1070                 return r;
1071         }
1072
1073         r = amdgpu_virt_enable_access_debugfs(adev);
1074         if (r < 0) {
1075                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1076                 return r;
1077         }
1078
1079         /* switch to the specific se/sh/cu */
1080         mutex_lock(&adev->grbm_idx_mutex);
1081         amdgpu_gfx_select_se_sh(adev, se, sh, cu, 0);
1082
1083         x = 0;
1084         if (adev->gfx.funcs->read_wave_data)
1085                 adev->gfx.funcs->read_wave_data(adev, 0, simd, wave, data, &x);
1086
1087         amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0);
1088         mutex_unlock(&adev->grbm_idx_mutex);
1089
1090         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1091         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1092
1093         if (!x) {
1094                 amdgpu_virt_disable_access_debugfs(adev);
1095                 return -EINVAL;
1096         }
1097
1098         while (size && (offset < x * 4)) {
1099                 uint32_t value;
1100
1101                 value = data[offset >> 2];
1102                 r = put_user(value, (uint32_t *)buf);
1103                 if (r) {
1104                         amdgpu_virt_disable_access_debugfs(adev);
1105                         return r;
1106                 }
1107
1108                 result += 4;
1109                 buf += 4;
1110                 offset += 4;
1111                 size -= 4;
1112         }
1113
1114         amdgpu_virt_disable_access_debugfs(adev);
1115         return result;
1116 }
1117
1118 /** amdgpu_debugfs_gpr_read - Read wave gprs
1119  *
1120  * @f: open file handle
1121  * @buf: User buffer to store read data in
1122  * @size: Number of bytes to read
1123  * @pos:  Offset to seek to
1124  *
1125  * The offset being sought changes which wave that the status data
1126  * will be returned for.  The bits are used as follows:
1127  *
1128  * Bits 0..11:  Byte offset into data
1129  * Bits 12..19: SE selector
1130  * Bits 20..27: SH/SA selector
1131  * Bits 28..35: CU/{WGP+SIMD} selector
1132  * Bits 36..43: WAVE ID selector
1133  * Bits 37..44: SIMD ID selector
1134  * Bits 52..59: Thread selector
1135  * Bits 60..61: Bank selector (VGPR=0,SGPR=1)
1136  *
1137  * The return data comes from the SGPR or VGPR register bank for
1138  * the selected operational unit.
1139  */
1140 static ssize_t amdgpu_debugfs_gpr_read(struct file *f, char __user *buf,
1141                                         size_t size, loff_t *pos)
1142 {
1143         struct amdgpu_device *adev = f->f_inode->i_private;
1144         int r;
1145         ssize_t result = 0;
1146         uint32_t offset, se, sh, cu, wave, simd, thread, bank, *data;
1147
1148         if (size > 4096 || size & 3 || *pos & 3)
1149                 return -EINVAL;
1150
1151         /* decode offset */
1152         offset = (*pos & GENMASK_ULL(11, 0)) >> 2;
1153         se = (*pos & GENMASK_ULL(19, 12)) >> 12;
1154         sh = (*pos & GENMASK_ULL(27, 20)) >> 20;
1155         cu = (*pos & GENMASK_ULL(35, 28)) >> 28;
1156         wave = (*pos & GENMASK_ULL(43, 36)) >> 36;
1157         simd = (*pos & GENMASK_ULL(51, 44)) >> 44;
1158         thread = (*pos & GENMASK_ULL(59, 52)) >> 52;
1159         bank = (*pos & GENMASK_ULL(61, 60)) >> 60;
1160
1161         data = kcalloc(1024, sizeof(*data), GFP_KERNEL);
1162         if (!data)
1163                 return -ENOMEM;
1164
1165         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1166         if (r < 0)
1167                 goto err;
1168
1169         r = amdgpu_virt_enable_access_debugfs(adev);
1170         if (r < 0)
1171                 goto err;
1172
1173         /* switch to the specific se/sh/cu */
1174         mutex_lock(&adev->grbm_idx_mutex);
1175         amdgpu_gfx_select_se_sh(adev, se, sh, cu, 0);
1176
1177         if (bank == 0) {
1178                 if (adev->gfx.funcs->read_wave_vgprs)
1179                         adev->gfx.funcs->read_wave_vgprs(adev, 0, simd, wave, thread, offset, size>>2, data);
1180         } else {
1181                 if (adev->gfx.funcs->read_wave_sgprs)
1182                         adev->gfx.funcs->read_wave_sgprs(adev, 0, simd, wave, offset, size>>2, data);
1183         }
1184
1185         amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0);
1186         mutex_unlock(&adev->grbm_idx_mutex);
1187
1188         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1189         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1190
1191         while (size) {
1192                 uint32_t value;
1193
1194                 value = data[result >> 2];
1195                 r = put_user(value, (uint32_t *)buf);
1196                 if (r) {
1197                         amdgpu_virt_disable_access_debugfs(adev);
1198                         goto err;
1199                 }
1200
1201                 result += 4;
1202                 buf += 4;
1203                 size -= 4;
1204         }
1205
1206         kfree(data);
1207         amdgpu_virt_disable_access_debugfs(adev);
1208         return result;
1209
1210 err:
1211         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1212         kfree(data);
1213         return r;
1214 }
1215
1216 /**
1217  * amdgpu_debugfs_gfxoff_residency_read - Read GFXOFF residency
1218  *
1219  * @f: open file handle
1220  * @buf: User buffer to store read data in
1221  * @size: Number of bytes to read
1222  * @pos:  Offset to seek to
1223  *
1224  * Read the last residency value logged. It doesn't auto update, one needs to
1225  * stop logging before getting the current value.
1226  */
1227 static ssize_t amdgpu_debugfs_gfxoff_residency_read(struct file *f, char __user *buf,
1228                                                     size_t size, loff_t *pos)
1229 {
1230         struct amdgpu_device *adev = file_inode(f)->i_private;
1231         ssize_t result = 0;
1232         int r;
1233
1234         if (size & 0x3 || *pos & 0x3)
1235                 return -EINVAL;
1236
1237         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1238         if (r < 0) {
1239                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1240                 return r;
1241         }
1242
1243         while (size) {
1244                 uint32_t value;
1245
1246                 r = amdgpu_get_gfx_off_residency(adev, &value);
1247                 if (r)
1248                         goto out;
1249
1250                 r = put_user(value, (uint32_t *)buf);
1251                 if (r)
1252                         goto out;
1253
1254                 result += 4;
1255                 buf += 4;
1256                 *pos += 4;
1257                 size -= 4;
1258         }
1259
1260         r = result;
1261 out:
1262         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1263         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1264
1265         return r;
1266 }
1267
1268 /**
1269  * amdgpu_debugfs_gfxoff_residency_write - Log GFXOFF Residency
1270  *
1271  * @f: open file handle
1272  * @buf: User buffer to write data from
1273  * @size: Number of bytes to write
1274  * @pos:  Offset to seek to
1275  *
1276  * Write a 32-bit non-zero to start logging; write a 32-bit zero to stop
1277  */
1278 static ssize_t amdgpu_debugfs_gfxoff_residency_write(struct file *f, const char __user *buf,
1279                                                      size_t size, loff_t *pos)
1280 {
1281         struct amdgpu_device *adev = file_inode(f)->i_private;
1282         ssize_t result = 0;
1283         int r;
1284
1285         if (size & 0x3 || *pos & 0x3)
1286                 return -EINVAL;
1287
1288         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1289         if (r < 0) {
1290                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1291                 return r;
1292         }
1293
1294         while (size) {
1295                 u32 value;
1296
1297                 r = get_user(value, (uint32_t *)buf);
1298                 if (r)
1299                         goto out;
1300
1301                 amdgpu_set_gfx_off_residency(adev, value ? true : false);
1302
1303                 result += 4;
1304                 buf += 4;
1305                 *pos += 4;
1306                 size -= 4;
1307         }
1308
1309         r = result;
1310 out:
1311         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1312         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1313
1314         return r;
1315 }
1316
1317
1318 /**
1319  * amdgpu_debugfs_gfxoff_count_read - Read GFXOFF entry count
1320  *
1321  * @f: open file handle
1322  * @buf: User buffer to store read data in
1323  * @size: Number of bytes to read
1324  * @pos:  Offset to seek to
1325  */
1326 static ssize_t amdgpu_debugfs_gfxoff_count_read(struct file *f, char __user *buf,
1327                                                 size_t size, loff_t *pos)
1328 {
1329         struct amdgpu_device *adev = file_inode(f)->i_private;
1330         ssize_t result = 0;
1331         int r;
1332
1333         if (size & 0x3 || *pos & 0x3)
1334                 return -EINVAL;
1335
1336         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1337         if (r < 0) {
1338                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1339                 return r;
1340         }
1341
1342         while (size) {
1343                 u64 value = 0;
1344
1345                 r = amdgpu_get_gfx_off_entrycount(adev, &value);
1346                 if (r)
1347                         goto out;
1348
1349                 r = put_user(value, (u64 *)buf);
1350                 if (r)
1351                         goto out;
1352
1353                 result += 4;
1354                 buf += 4;
1355                 *pos += 4;
1356                 size -= 4;
1357         }
1358
1359         r = result;
1360 out:
1361         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1362         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1363
1364         return r;
1365 }
1366
1367 /**
1368  * amdgpu_debugfs_gfxoff_write - Enable/disable GFXOFF
1369  *
1370  * @f: open file handle
1371  * @buf: User buffer to write data from
1372  * @size: Number of bytes to write
1373  * @pos:  Offset to seek to
1374  *
1375  * Write a 32-bit zero to disable or a 32-bit non-zero to enable
1376  */
1377 static ssize_t amdgpu_debugfs_gfxoff_write(struct file *f, const char __user *buf,
1378                                          size_t size, loff_t *pos)
1379 {
1380         struct amdgpu_device *adev = file_inode(f)->i_private;
1381         ssize_t result = 0;
1382         int r;
1383
1384         if (size & 0x3 || *pos & 0x3)
1385                 return -EINVAL;
1386
1387         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1388         if (r < 0) {
1389                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1390                 return r;
1391         }
1392
1393         while (size) {
1394                 uint32_t value;
1395
1396                 r = get_user(value, (uint32_t *)buf);
1397                 if (r)
1398                         goto out;
1399
1400                 amdgpu_gfx_off_ctrl(adev, value ? true : false);
1401
1402                 result += 4;
1403                 buf += 4;
1404                 *pos += 4;
1405                 size -= 4;
1406         }
1407
1408         r = result;
1409 out:
1410         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1411         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1412
1413         return r;
1414 }
1415
1416
1417 /**
1418  * amdgpu_debugfs_gfxoff_read - read gfxoff status
1419  *
1420  * @f: open file handle
1421  * @buf: User buffer to store read data in
1422  * @size: Number of bytes to read
1423  * @pos:  Offset to seek to
1424  */
1425 static ssize_t amdgpu_debugfs_gfxoff_read(struct file *f, char __user *buf,
1426                                          size_t size, loff_t *pos)
1427 {
1428         struct amdgpu_device *adev = file_inode(f)->i_private;
1429         ssize_t result = 0;
1430         int r;
1431
1432         if (size & 0x3 || *pos & 0x3)
1433                 return -EINVAL;
1434
1435         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1436         if (r < 0) {
1437                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1438                 return r;
1439         }
1440
1441         while (size) {
1442                 u32 value = adev->gfx.gfx_off_state;
1443
1444                 r = put_user(value, (u32 *)buf);
1445                 if (r)
1446                         goto out;
1447
1448                 result += 4;
1449                 buf += 4;
1450                 *pos += 4;
1451                 size -= 4;
1452         }
1453
1454         r = result;
1455 out:
1456         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1457         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1458
1459         return r;
1460 }
1461
1462 static ssize_t amdgpu_debugfs_gfxoff_status_read(struct file *f, char __user *buf,
1463                                                  size_t size, loff_t *pos)
1464 {
1465         struct amdgpu_device *adev = file_inode(f)->i_private;
1466         ssize_t result = 0;
1467         int r;
1468
1469         if (size & 0x3 || *pos & 0x3)
1470                 return -EINVAL;
1471
1472         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1473         if (r < 0) {
1474                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1475                 return r;
1476         }
1477
1478         while (size) {
1479                 u32 value;
1480
1481                 r = amdgpu_get_gfx_off_status(adev, &value);
1482                 if (r)
1483                         goto out;
1484
1485                 r = put_user(value, (u32 *)buf);
1486                 if (r)
1487                         goto out;
1488
1489                 result += 4;
1490                 buf += 4;
1491                 *pos += 4;
1492                 size -= 4;
1493         }
1494
1495         r = result;
1496 out:
1497         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1498         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1499
1500         return r;
1501 }
1502
1503 static const struct file_operations amdgpu_debugfs_regs2_fops = {
1504         .owner = THIS_MODULE,
1505         .unlocked_ioctl = amdgpu_debugfs_regs2_ioctl,
1506         .read = amdgpu_debugfs_regs2_read,
1507         .write = amdgpu_debugfs_regs2_write,
1508         .open = amdgpu_debugfs_regs2_open,
1509         .release = amdgpu_debugfs_regs2_release,
1510         .llseek = default_llseek
1511 };
1512
1513 static const struct file_operations amdgpu_debugfs_gprwave_fops = {
1514         .owner = THIS_MODULE,
1515         .unlocked_ioctl = amdgpu_debugfs_gprwave_ioctl,
1516         .read = amdgpu_debugfs_gprwave_read,
1517         .open = amdgpu_debugfs_gprwave_open,
1518         .release = amdgpu_debugfs_gprwave_release,
1519         .llseek = default_llseek
1520 };
1521
1522 static const struct file_operations amdgpu_debugfs_regs_fops = {
1523         .owner = THIS_MODULE,
1524         .read = amdgpu_debugfs_regs_read,
1525         .write = amdgpu_debugfs_regs_write,
1526         .llseek = default_llseek
1527 };
1528 static const struct file_operations amdgpu_debugfs_regs_didt_fops = {
1529         .owner = THIS_MODULE,
1530         .read = amdgpu_debugfs_regs_didt_read,
1531         .write = amdgpu_debugfs_regs_didt_write,
1532         .llseek = default_llseek
1533 };
1534 static const struct file_operations amdgpu_debugfs_regs_pcie_fops = {
1535         .owner = THIS_MODULE,
1536         .read = amdgpu_debugfs_regs_pcie_read,
1537         .write = amdgpu_debugfs_regs_pcie_write,
1538         .llseek = default_llseek
1539 };
1540 static const struct file_operations amdgpu_debugfs_regs_smc_fops = {
1541         .owner = THIS_MODULE,
1542         .read = amdgpu_debugfs_regs_smc_read,
1543         .write = amdgpu_debugfs_regs_smc_write,
1544         .llseek = default_llseek
1545 };
1546
1547 static const struct file_operations amdgpu_debugfs_gca_config_fops = {
1548         .owner = THIS_MODULE,
1549         .read = amdgpu_debugfs_gca_config_read,
1550         .llseek = default_llseek
1551 };
1552
1553 static const struct file_operations amdgpu_debugfs_sensors_fops = {
1554         .owner = THIS_MODULE,
1555         .read = amdgpu_debugfs_sensor_read,
1556         .llseek = default_llseek
1557 };
1558
1559 static const struct file_operations amdgpu_debugfs_wave_fops = {
1560         .owner = THIS_MODULE,
1561         .read = amdgpu_debugfs_wave_read,
1562         .llseek = default_llseek
1563 };
1564 static const struct file_operations amdgpu_debugfs_gpr_fops = {
1565         .owner = THIS_MODULE,
1566         .read = amdgpu_debugfs_gpr_read,
1567         .llseek = default_llseek
1568 };
1569
1570 static const struct file_operations amdgpu_debugfs_gfxoff_fops = {
1571         .owner = THIS_MODULE,
1572         .read = amdgpu_debugfs_gfxoff_read,
1573         .write = amdgpu_debugfs_gfxoff_write,
1574         .llseek = default_llseek
1575 };
1576
1577 static const struct file_operations amdgpu_debugfs_gfxoff_status_fops = {
1578         .owner = THIS_MODULE,
1579         .read = amdgpu_debugfs_gfxoff_status_read,
1580         .llseek = default_llseek
1581 };
1582
1583 static const struct file_operations amdgpu_debugfs_gfxoff_count_fops = {
1584         .owner = THIS_MODULE,
1585         .read = amdgpu_debugfs_gfxoff_count_read,
1586         .llseek = default_llseek
1587 };
1588
1589 static const struct file_operations amdgpu_debugfs_gfxoff_residency_fops = {
1590         .owner = THIS_MODULE,
1591         .read = amdgpu_debugfs_gfxoff_residency_read,
1592         .write = amdgpu_debugfs_gfxoff_residency_write,
1593         .llseek = default_llseek
1594 };
1595
1596 static const struct file_operations *debugfs_regs[] = {
1597         &amdgpu_debugfs_regs_fops,
1598         &amdgpu_debugfs_regs2_fops,
1599         &amdgpu_debugfs_gprwave_fops,
1600         &amdgpu_debugfs_regs_didt_fops,
1601         &amdgpu_debugfs_regs_pcie_fops,
1602         &amdgpu_debugfs_regs_smc_fops,
1603         &amdgpu_debugfs_gca_config_fops,
1604         &amdgpu_debugfs_sensors_fops,
1605         &amdgpu_debugfs_wave_fops,
1606         &amdgpu_debugfs_gpr_fops,
1607         &amdgpu_debugfs_gfxoff_fops,
1608         &amdgpu_debugfs_gfxoff_status_fops,
1609         &amdgpu_debugfs_gfxoff_count_fops,
1610         &amdgpu_debugfs_gfxoff_residency_fops,
1611 };
1612
1613 static const char * const debugfs_regs_names[] = {
1614         "amdgpu_regs",
1615         "amdgpu_regs2",
1616         "amdgpu_gprwave",
1617         "amdgpu_regs_didt",
1618         "amdgpu_regs_pcie",
1619         "amdgpu_regs_smc",
1620         "amdgpu_gca_config",
1621         "amdgpu_sensors",
1622         "amdgpu_wave",
1623         "amdgpu_gpr",
1624         "amdgpu_gfxoff",
1625         "amdgpu_gfxoff_status",
1626         "amdgpu_gfxoff_count",
1627         "amdgpu_gfxoff_residency",
1628 };
1629
1630 /**
1631  * amdgpu_debugfs_regs_init -   Initialize debugfs entries that provide
1632  *                              register access.
1633  *
1634  * @adev: The device to attach the debugfs entries to
1635  */
1636 int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
1637 {
1638         struct drm_minor *minor = adev_to_drm(adev)->primary;
1639         struct dentry *ent, *root = minor->debugfs_root;
1640         unsigned int i;
1641
1642         for (i = 0; i < ARRAY_SIZE(debugfs_regs); i++) {
1643                 ent = debugfs_create_file(debugfs_regs_names[i],
1644                                           S_IFREG | 0444, root,
1645                                           adev, debugfs_regs[i]);
1646                 if (!i && !IS_ERR_OR_NULL(ent))
1647                         i_size_write(ent->d_inode, adev->rmmio_size);
1648         }
1649
1650         return 0;
1651 }
1652
1653 static int amdgpu_debugfs_test_ib_show(struct seq_file *m, void *unused)
1654 {
1655         struct amdgpu_device *adev = m->private;
1656         struct drm_device *dev = adev_to_drm(adev);
1657         int r = 0, i;
1658
1659         r = pm_runtime_get_sync(dev->dev);
1660         if (r < 0) {
1661                 pm_runtime_put_autosuspend(dev->dev);
1662                 return r;
1663         }
1664
1665         /* Avoid accidently unparking the sched thread during GPU reset */
1666         r = down_write_killable(&adev->reset_domain->sem);
1667         if (r)
1668                 return r;
1669
1670         /* hold on the scheduler */
1671         for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
1672                 struct amdgpu_ring *ring = adev->rings[i];
1673
1674                 if (!ring || !ring->sched.thread)
1675                         continue;
1676                 kthread_park(ring->sched.thread);
1677         }
1678
1679         seq_puts(m, "run ib test:\n");
1680         r = amdgpu_ib_ring_tests(adev);
1681         if (r)
1682                 seq_printf(m, "ib ring tests failed (%d).\n", r);
1683         else
1684                 seq_puts(m, "ib ring tests passed.\n");
1685
1686         /* go on the scheduler */
1687         for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
1688                 struct amdgpu_ring *ring = adev->rings[i];
1689
1690                 if (!ring || !ring->sched.thread)
1691                         continue;
1692                 kthread_unpark(ring->sched.thread);
1693         }
1694
1695         up_write(&adev->reset_domain->sem);
1696
1697         pm_runtime_mark_last_busy(dev->dev);
1698         pm_runtime_put_autosuspend(dev->dev);
1699
1700         return 0;
1701 }
1702
1703 static int amdgpu_debugfs_evict_vram(void *data, u64 *val)
1704 {
1705         struct amdgpu_device *adev = (struct amdgpu_device *)data;
1706         struct drm_device *dev = adev_to_drm(adev);
1707         int r;
1708
1709         r = pm_runtime_get_sync(dev->dev);
1710         if (r < 0) {
1711                 pm_runtime_put_autosuspend(dev->dev);
1712                 return r;
1713         }
1714
1715         *val = amdgpu_ttm_evict_resources(adev, TTM_PL_VRAM);
1716
1717         pm_runtime_mark_last_busy(dev->dev);
1718         pm_runtime_put_autosuspend(dev->dev);
1719
1720         return 0;
1721 }
1722
1723
1724 static int amdgpu_debugfs_evict_gtt(void *data, u64 *val)
1725 {
1726         struct amdgpu_device *adev = (struct amdgpu_device *)data;
1727         struct drm_device *dev = adev_to_drm(adev);
1728         int r;
1729
1730         r = pm_runtime_get_sync(dev->dev);
1731         if (r < 0) {
1732                 pm_runtime_put_autosuspend(dev->dev);
1733                 return r;
1734         }
1735
1736         *val = amdgpu_ttm_evict_resources(adev, TTM_PL_TT);
1737
1738         pm_runtime_mark_last_busy(dev->dev);
1739         pm_runtime_put_autosuspend(dev->dev);
1740
1741         return 0;
1742 }
1743
1744 static int amdgpu_debugfs_benchmark(void *data, u64 val)
1745 {
1746         struct amdgpu_device *adev = (struct amdgpu_device *)data;
1747         struct drm_device *dev = adev_to_drm(adev);
1748         int r;
1749
1750         r = pm_runtime_get_sync(dev->dev);
1751         if (r < 0) {
1752                 pm_runtime_put_autosuspend(dev->dev);
1753                 return r;
1754         }
1755
1756         r = amdgpu_benchmark(adev, val);
1757
1758         pm_runtime_mark_last_busy(dev->dev);
1759         pm_runtime_put_autosuspend(dev->dev);
1760
1761         return r;
1762 }
1763
1764 static int amdgpu_debugfs_vm_info_show(struct seq_file *m, void *unused)
1765 {
1766         struct amdgpu_device *adev = m->private;
1767         struct drm_device *dev = adev_to_drm(adev);
1768         struct drm_file *file;
1769         int r;
1770
1771         r = mutex_lock_interruptible(&dev->filelist_mutex);
1772         if (r)
1773                 return r;
1774
1775         list_for_each_entry(file, &dev->filelist, lhead) {
1776                 struct amdgpu_fpriv *fpriv = file->driver_priv;
1777                 struct amdgpu_vm *vm = &fpriv->vm;
1778
1779                 seq_printf(m, "pid:%d\tProcess:%s ----------\n",
1780                                 vm->task_info.pid, vm->task_info.process_name);
1781                 r = amdgpu_bo_reserve(vm->root.bo, true);
1782                 if (r)
1783                         break;
1784                 amdgpu_debugfs_vm_bo_info(vm, m);
1785                 amdgpu_bo_unreserve(vm->root.bo);
1786         }
1787
1788         mutex_unlock(&dev->filelist_mutex);
1789
1790         return r;
1791 }
1792
1793 DEFINE_SHOW_ATTRIBUTE(amdgpu_debugfs_test_ib);
1794 DEFINE_SHOW_ATTRIBUTE(amdgpu_debugfs_vm_info);
1795 DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_evict_vram_fops, amdgpu_debugfs_evict_vram,
1796                          NULL, "%lld\n");
1797 DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_evict_gtt_fops, amdgpu_debugfs_evict_gtt,
1798                          NULL, "%lld\n");
1799 DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_benchmark_fops, NULL, amdgpu_debugfs_benchmark,
1800                          "%lld\n");
1801
1802 static void amdgpu_ib_preempt_fences_swap(struct amdgpu_ring *ring,
1803                                           struct dma_fence **fences)
1804 {
1805         struct amdgpu_fence_driver *drv = &ring->fence_drv;
1806         uint32_t sync_seq, last_seq;
1807
1808         last_seq = atomic_read(&ring->fence_drv.last_seq);
1809         sync_seq = ring->fence_drv.sync_seq;
1810
1811         last_seq &= drv->num_fences_mask;
1812         sync_seq &= drv->num_fences_mask;
1813
1814         do {
1815                 struct dma_fence *fence, **ptr;
1816
1817                 ++last_seq;
1818                 last_seq &= drv->num_fences_mask;
1819                 ptr = &drv->fences[last_seq];
1820
1821                 fence = rcu_dereference_protected(*ptr, 1);
1822                 RCU_INIT_POINTER(*ptr, NULL);
1823
1824                 if (!fence)
1825                         continue;
1826
1827                 fences[last_seq] = fence;
1828
1829         } while (last_seq != sync_seq);
1830 }
1831
1832 static void amdgpu_ib_preempt_signal_fences(struct dma_fence **fences,
1833                                             int length)
1834 {
1835         int i;
1836         struct dma_fence *fence;
1837
1838         for (i = 0; i < length; i++) {
1839                 fence = fences[i];
1840                 if (!fence)
1841                         continue;
1842                 dma_fence_signal(fence);
1843                 dma_fence_put(fence);
1844         }
1845 }
1846
1847 static void amdgpu_ib_preempt_job_recovery(struct drm_gpu_scheduler *sched)
1848 {
1849         struct drm_sched_job *s_job;
1850         struct dma_fence *fence;
1851
1852         spin_lock(&sched->job_list_lock);
1853         list_for_each_entry(s_job, &sched->pending_list, list) {
1854                 fence = sched->ops->run_job(s_job);
1855                 dma_fence_put(fence);
1856         }
1857         spin_unlock(&sched->job_list_lock);
1858 }
1859
1860 static void amdgpu_ib_preempt_mark_partial_job(struct amdgpu_ring *ring)
1861 {
1862         struct amdgpu_job *job;
1863         struct drm_sched_job *s_job, *tmp;
1864         uint32_t preempt_seq;
1865         struct dma_fence *fence, **ptr;
1866         struct amdgpu_fence_driver *drv = &ring->fence_drv;
1867         struct drm_gpu_scheduler *sched = &ring->sched;
1868         bool preempted = true;
1869
1870         if (ring->funcs->type != AMDGPU_RING_TYPE_GFX)
1871                 return;
1872
1873         preempt_seq = le32_to_cpu(*(drv->cpu_addr + 2));
1874         if (preempt_seq <= atomic_read(&drv->last_seq)) {
1875                 preempted = false;
1876                 goto no_preempt;
1877         }
1878
1879         preempt_seq &= drv->num_fences_mask;
1880         ptr = &drv->fences[preempt_seq];
1881         fence = rcu_dereference_protected(*ptr, 1);
1882
1883 no_preempt:
1884         spin_lock(&sched->job_list_lock);
1885         list_for_each_entry_safe(s_job, tmp, &sched->pending_list, list) {
1886                 if (dma_fence_is_signaled(&s_job->s_fence->finished)) {
1887                         /* remove job from ring_mirror_list */
1888                         list_del_init(&s_job->list);
1889                         sched->ops->free_job(s_job);
1890                         continue;
1891                 }
1892                 job = to_amdgpu_job(s_job);
1893                 if (preempted && (&job->hw_fence) == fence)
1894                         /* mark the job as preempted */
1895                         job->preemption_status |= AMDGPU_IB_PREEMPTED;
1896         }
1897         spin_unlock(&sched->job_list_lock);
1898 }
1899
1900 static int amdgpu_debugfs_ib_preempt(void *data, u64 val)
1901 {
1902         int r, length;
1903         struct amdgpu_ring *ring;
1904         struct dma_fence **fences = NULL;
1905         struct amdgpu_device *adev = (struct amdgpu_device *)data;
1906
1907         if (val >= AMDGPU_MAX_RINGS)
1908                 return -EINVAL;
1909
1910         ring = adev->rings[val];
1911
1912         if (!ring || !ring->funcs->preempt_ib || !ring->sched.thread)
1913                 return -EINVAL;
1914
1915         /* the last preemption failed */
1916         if (ring->trail_seq != le32_to_cpu(*ring->trail_fence_cpu_addr))
1917                 return -EBUSY;
1918
1919         length = ring->fence_drv.num_fences_mask + 1;
1920         fences = kcalloc(length, sizeof(void *), GFP_KERNEL);
1921         if (!fences)
1922                 return -ENOMEM;
1923
1924         /* Avoid accidently unparking the sched thread during GPU reset */
1925         r = down_read_killable(&adev->reset_domain->sem);
1926         if (r)
1927                 goto pro_end;
1928
1929         /* stop the scheduler */
1930         kthread_park(ring->sched.thread);
1931
1932         /* preempt the IB */
1933         r = amdgpu_ring_preempt_ib(ring);
1934         if (r) {
1935                 DRM_WARN("failed to preempt ring %d\n", ring->idx);
1936                 goto failure;
1937         }
1938
1939         amdgpu_fence_process(ring);
1940
1941         if (atomic_read(&ring->fence_drv.last_seq) !=
1942             ring->fence_drv.sync_seq) {
1943                 DRM_INFO("ring %d was preempted\n", ring->idx);
1944
1945                 amdgpu_ib_preempt_mark_partial_job(ring);
1946
1947                 /* swap out the old fences */
1948                 amdgpu_ib_preempt_fences_swap(ring, fences);
1949
1950                 amdgpu_fence_driver_force_completion(ring);
1951
1952                 /* resubmit unfinished jobs */
1953                 amdgpu_ib_preempt_job_recovery(&ring->sched);
1954
1955                 /* wait for jobs finished */
1956                 amdgpu_fence_wait_empty(ring);
1957
1958                 /* signal the old fences */
1959                 amdgpu_ib_preempt_signal_fences(fences, length);
1960         }
1961
1962 failure:
1963         /* restart the scheduler */
1964         kthread_unpark(ring->sched.thread);
1965
1966         up_read(&adev->reset_domain->sem);
1967
1968 pro_end:
1969         kfree(fences);
1970
1971         return r;
1972 }
1973
1974 static int amdgpu_debugfs_sclk_set(void *data, u64 val)
1975 {
1976         int ret = 0;
1977         uint32_t max_freq, min_freq;
1978         struct amdgpu_device *adev = (struct amdgpu_device *)data;
1979
1980         if (amdgpu_sriov_vf(adev) && !amdgpu_sriov_is_pp_one_vf(adev))
1981                 return -EINVAL;
1982
1983         ret = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1984         if (ret < 0) {
1985                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1986                 return ret;
1987         }
1988
1989         ret = amdgpu_dpm_get_dpm_freq_range(adev, PP_SCLK, &min_freq, &max_freq);
1990         if (ret == -EOPNOTSUPP) {
1991                 ret = 0;
1992                 goto out;
1993         }
1994         if (ret || val > max_freq || val < min_freq) {
1995                 ret = -EINVAL;
1996                 goto out;
1997         }
1998
1999         ret = amdgpu_dpm_set_soft_freq_range(adev, PP_SCLK, (uint32_t)val, (uint32_t)val);
2000         if (ret)
2001                 ret = -EINVAL;
2002
2003 out:
2004         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
2005         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
2006
2007         return ret;
2008 }
2009
2010 DEFINE_DEBUGFS_ATTRIBUTE(fops_ib_preempt, NULL,
2011                         amdgpu_debugfs_ib_preempt, "%llu\n");
2012
2013 DEFINE_DEBUGFS_ATTRIBUTE(fops_sclk_set, NULL,
2014                         amdgpu_debugfs_sclk_set, "%llu\n");
2015
2016 static ssize_t amdgpu_reset_dump_register_list_read(struct file *f,
2017                                 char __user *buf, size_t size, loff_t *pos)
2018 {
2019         struct amdgpu_device *adev = (struct amdgpu_device *)file_inode(f)->i_private;
2020         char reg_offset[12];
2021         int i, ret, len = 0;
2022
2023         if (*pos)
2024                 return 0;
2025
2026         memset(reg_offset, 0, 12);
2027         ret = down_read_killable(&adev->reset_domain->sem);
2028         if (ret)
2029                 return ret;
2030
2031         for (i = 0; i < adev->reset_info.num_regs; i++) {
2032                 sprintf(reg_offset, "0x%x\n", adev->reset_info.reset_dump_reg_list[i]);
2033                 up_read(&adev->reset_domain->sem);
2034                 if (copy_to_user(buf + len, reg_offset, strlen(reg_offset)))
2035                         return -EFAULT;
2036
2037                 len += strlen(reg_offset);
2038                 ret = down_read_killable(&adev->reset_domain->sem);
2039                 if (ret)
2040                         return ret;
2041         }
2042
2043         up_read(&adev->reset_domain->sem);
2044         *pos += len;
2045
2046         return len;
2047 }
2048
2049 static ssize_t amdgpu_reset_dump_register_list_write(struct file *f,
2050                         const char __user *buf, size_t size, loff_t *pos)
2051 {
2052         struct amdgpu_device *adev = (struct amdgpu_device *)file_inode(f)->i_private;
2053         char reg_offset[11];
2054         uint32_t *new = NULL, *tmp = NULL;
2055         int ret, i = 0, len = 0;
2056
2057         do {
2058                 memset(reg_offset, 0, 11);
2059                 if (copy_from_user(reg_offset, buf + len,
2060                                         min(10, ((int)size-len)))) {
2061                         ret = -EFAULT;
2062                         goto error_free;
2063                 }
2064
2065                 new = krealloc_array(tmp, i + 1, sizeof(uint32_t), GFP_KERNEL);
2066                 if (!new) {
2067                         ret = -ENOMEM;
2068                         goto error_free;
2069                 }
2070                 tmp = new;
2071                 if (sscanf(reg_offset, "%X %n", &tmp[i], &ret) != 1) {
2072                         ret = -EINVAL;
2073                         goto error_free;
2074                 }
2075
2076                 len += ret;
2077                 i++;
2078         } while (len < size);
2079
2080         new = kmalloc_array(i, sizeof(uint32_t), GFP_KERNEL);
2081         if (!new) {
2082                 ret = -ENOMEM;
2083                 goto error_free;
2084         }
2085         ret = down_write_killable(&adev->reset_domain->sem);
2086         if (ret)
2087                 goto error_free;
2088
2089         swap(adev->reset_info.reset_dump_reg_list, tmp);
2090         swap(adev->reset_info.reset_dump_reg_value, new);
2091         adev->reset_info.num_regs = i;
2092         up_write(&adev->reset_domain->sem);
2093         ret = size;
2094
2095 error_free:
2096         if (tmp != new)
2097                 kfree(tmp);
2098         kfree(new);
2099         return ret;
2100 }
2101
2102 static const struct file_operations amdgpu_reset_dump_register_list = {
2103         .owner = THIS_MODULE,
2104         .read = amdgpu_reset_dump_register_list_read,
2105         .write = amdgpu_reset_dump_register_list_write,
2106         .llseek = default_llseek
2107 };
2108
2109 int amdgpu_debugfs_init(struct amdgpu_device *adev)
2110 {
2111         struct dentry *root = adev_to_drm(adev)->primary->debugfs_root;
2112         struct dentry *ent;
2113         int r, i;
2114
2115         if (!debugfs_initialized())
2116                 return 0;
2117
2118         debugfs_create_x32("amdgpu_smu_debug", 0600, root,
2119                            &adev->pm.smu_debug_mask);
2120
2121         ent = debugfs_create_file("amdgpu_preempt_ib", 0600, root, adev,
2122                                   &fops_ib_preempt);
2123         if (IS_ERR(ent)) {
2124                 DRM_ERROR("unable to create amdgpu_preempt_ib debugsfs file\n");
2125                 return PTR_ERR(ent);
2126         }
2127
2128         ent = debugfs_create_file("amdgpu_force_sclk", 0200, root, adev,
2129                                   &fops_sclk_set);
2130         if (IS_ERR(ent)) {
2131                 DRM_ERROR("unable to create amdgpu_set_sclk debugsfs file\n");
2132                 return PTR_ERR(ent);
2133         }
2134
2135         /* Register debugfs entries for amdgpu_ttm */
2136         amdgpu_ttm_debugfs_init(adev);
2137         amdgpu_debugfs_pm_init(adev);
2138         amdgpu_debugfs_sa_init(adev);
2139         amdgpu_debugfs_fence_init(adev);
2140         amdgpu_debugfs_gem_init(adev);
2141
2142         r = amdgpu_debugfs_regs_init(adev);
2143         if (r)
2144                 DRM_ERROR("registering register debugfs failed (%d).\n", r);
2145
2146         amdgpu_debugfs_firmware_init(adev);
2147         amdgpu_ta_if_debugfs_init(adev);
2148
2149 #if defined(CONFIG_DRM_AMD_DC)
2150         if (adev->dc_enabled)
2151                 dtn_debugfs_init(adev);
2152 #endif
2153
2154         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
2155                 struct amdgpu_ring *ring = adev->rings[i];
2156
2157                 if (!ring)
2158                         continue;
2159
2160                 amdgpu_debugfs_ring_init(adev, ring);
2161         }
2162
2163         for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
2164                 if (!amdgpu_vcnfw_log)
2165                         break;
2166
2167                 if (adev->vcn.harvest_config & (1 << i))
2168                         continue;
2169
2170                 amdgpu_debugfs_vcn_fwlog_init(adev, i, &adev->vcn.inst[i]);
2171         }
2172
2173         amdgpu_ras_debugfs_create_all(adev);
2174         amdgpu_rap_debugfs_init(adev);
2175         amdgpu_securedisplay_debugfs_init(adev);
2176         amdgpu_fw_attestation_debugfs_init(adev);
2177
2178         debugfs_create_file("amdgpu_evict_vram", 0444, root, adev,
2179                             &amdgpu_evict_vram_fops);
2180         debugfs_create_file("amdgpu_evict_gtt", 0444, root, adev,
2181                             &amdgpu_evict_gtt_fops);
2182         debugfs_create_file("amdgpu_test_ib", 0444, root, adev,
2183                             &amdgpu_debugfs_test_ib_fops);
2184         debugfs_create_file("amdgpu_vm_info", 0444, root, adev,
2185                             &amdgpu_debugfs_vm_info_fops);
2186         debugfs_create_file("amdgpu_benchmark", 0200, root, adev,
2187                             &amdgpu_benchmark_fops);
2188         debugfs_create_file("amdgpu_reset_dump_register_list", 0644, root, adev,
2189                             &amdgpu_reset_dump_register_list);
2190
2191         adev->debugfs_vbios_blob.data = adev->bios;
2192         adev->debugfs_vbios_blob.size = adev->bios_size;
2193         debugfs_create_blob("amdgpu_vbios", 0444, root,
2194                             &adev->debugfs_vbios_blob);
2195
2196         adev->debugfs_discovery_blob.data = adev->mman.discovery_bin;
2197         adev->debugfs_discovery_blob.size = adev->mman.discovery_tmr_size;
2198         debugfs_create_blob("amdgpu_discovery", 0444, root,
2199                             &adev->debugfs_discovery_blob);
2200
2201         return 0;
2202 }
2203
2204 #else
2205 int amdgpu_debugfs_init(struct amdgpu_device *adev)
2206 {
2207         return 0;
2208 }
2209 int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
2210 {
2211         return 0;
2212 }
2213 #endif