drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c

   1 /*
   2  * Copyright 2014 Advanced Micro Devices, Inc.
   3  * Copyright 2008 Red Hat Inc.
   4  * Copyright 2009 Jerome Glisse.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22  * OTHER DEALINGS IN THE SOFTWARE.
  23  *
  24  */
  25
  26 #include <linux/firmware.h>
  27 #include <linux/pm_runtime.h>
  28
  29 #include "amdgpu.h"
  30 #include "amdgpu_gfx.h"
  31 #include "amdgpu_rlc.h"
  32 #include "amdgpu_ras.h"
  33 #include "amdgpu_reset.h"
  34 #include "amdgpu_xcp.h"
  35 #include "amdgpu_xgmi.h"
  36
  37 /* delay 0.1 second to enable gfx off feature */
  38 #define GFX_OFF_DELAY_ENABLE         msecs_to_jiffies(100)
  39
  40 #define GFX_OFF_NO_DELAY 0
  41
  42 /*
  43  * GPU GFX IP block helpers function.
  44  */
  45
  46 int amdgpu_gfx_mec_queue_to_bit(struct amdgpu_device *adev, int mec,
  47                                 int pipe, int queue)
  48 {
  49         int bit = 0;
  50
  51         bit += mec * adev->gfx.mec.num_pipe_per_mec
  52                 * adev->gfx.mec.num_queue_per_pipe;
  53         bit += pipe * adev->gfx.mec.num_queue_per_pipe;
  54         bit += queue;
  55
  56         return bit;
  57 }
  58
  59 void amdgpu_queue_mask_bit_to_mec_queue(struct amdgpu_device *adev, int bit,
  60                                  int *mec, int *pipe, int *queue)
  61 {
  62         *queue = bit % adev->gfx.mec.num_queue_per_pipe;
  63         *pipe = (bit / adev->gfx.mec.num_queue_per_pipe)
  64                 % adev->gfx.mec.num_pipe_per_mec;
  65         *mec = (bit / adev->gfx.mec.num_queue_per_pipe)
  66                / adev->gfx.mec.num_pipe_per_mec;
  67
  68 }
  69
  70 bool amdgpu_gfx_is_mec_queue_enabled(struct amdgpu_device *adev,
  71                                      int xcc_id, int mec, int pipe, int queue)
  72 {
  73         return test_bit(amdgpu_gfx_mec_queue_to_bit(adev, mec, pipe, queue),
  74                         adev->gfx.mec_bitmap[xcc_id].queue_bitmap);
  75 }
  76
  77 int amdgpu_gfx_me_queue_to_bit(struct amdgpu_device *adev,
  78                                int me, int pipe, int queue)
  79 {
  80         int bit = 0;
  81
  82         bit += me * adev->gfx.me.num_pipe_per_me
  83                 * adev->gfx.me.num_queue_per_pipe;
  84         bit += pipe * adev->gfx.me.num_queue_per_pipe;
  85         bit += queue;
  86
  87         return bit;
  88 }
  89
  90 bool amdgpu_gfx_is_me_queue_enabled(struct amdgpu_device *adev,
  91                                     int me, int pipe, int queue)
  92 {
  93         return test_bit(amdgpu_gfx_me_queue_to_bit(adev, me, pipe, queue),
  94                         adev->gfx.me.queue_bitmap);
  95 }
  96
  97 /**
  98  * amdgpu_gfx_parse_disable_cu - Parse the disable_cu module parameter
  99  *
 100  * @mask: array in which the per-shader array disable masks will be stored
 101  * @max_se: number of SEs
 102  * @max_sh: number of SHs
 103  *
 104  * The bitmask of CUs to be disabled in the shader array determined by se and
 105  * sh is stored in mask[se * max_sh + sh].
 106  */
 107 void amdgpu_gfx_parse_disable_cu(unsigned int *mask, unsigned int max_se, unsigned int max_sh)
 108 {
 109         unsigned int se, sh, cu;
 110         const char *p;
 111
 112         memset(mask, 0, sizeof(*mask) * max_se * max_sh);
 113
 114         if (!amdgpu_disable_cu || !*amdgpu_disable_cu)
 115                 return;
 116
 117         p = amdgpu_disable_cu;
 118         for (;;) {
 119                 char *next;
 120                 int ret = sscanf(p, "%u.%u.%u", &se, &sh, &cu);
 121
 122                 if (ret < 3) {
 123                         DRM_ERROR("amdgpu: could not parse disable_cu\n");
 124                         return;
 125                 }
 126
 127                 if (se < max_se && sh < max_sh && cu < 16) {
 128                         DRM_INFO("amdgpu: disabling CU %u.%u.%u\n", se, sh, cu);
 129                         mask[se * max_sh + sh] |= 1u << cu;
 130                 } else {
 131                         DRM_ERROR("amdgpu: disable_cu %u.%u.%u is out of range\n",
 132                                   se, sh, cu);
 133                 }
 134
 135                 next = strchr(p, ',');
 136                 if (!next)
 137                         break;
 138                 p = next + 1;
 139         }
 140 }
 141
 142 static bool amdgpu_gfx_is_graphics_multipipe_capable(struct amdgpu_device *adev)
 143 {
 144         return amdgpu_async_gfx_ring && adev->gfx.me.num_pipe_per_me > 1;
 145 }
 146
 147 static bool amdgpu_gfx_is_compute_multipipe_capable(struct amdgpu_device *adev)
 148 {
 149         if (amdgpu_compute_multipipe != -1) {
 150                 DRM_INFO("amdgpu: forcing compute pipe policy %d\n",
 151                          amdgpu_compute_multipipe);
 152                 return amdgpu_compute_multipipe == 1;
 153         }
 154
 155         if (amdgpu_ip_version(adev, GC_HWIP, 0) > IP_VERSION(9, 0, 0))
 156                 return true;
 157
 158         /* FIXME: spreading the queues across pipes causes perf regressions
 159          * on POLARIS11 compute workloads */
 160         if (adev->asic_type == CHIP_POLARIS11)
 161                 return false;
 162
 163         return adev->gfx.mec.num_mec > 1;
 164 }
 165
 166 bool amdgpu_gfx_is_high_priority_graphics_queue(struct amdgpu_device *adev,
 167                                                 struct amdgpu_ring *ring)
 168 {
 169         int queue = ring->queue;
 170         int pipe = ring->pipe;
 171
 172         /* Policy: use pipe1 queue0 as high priority graphics queue if we
 173          * have more than one gfx pipe.
 174          */
 175         if (amdgpu_gfx_is_graphics_multipipe_capable(adev) &&
 176             adev->gfx.num_gfx_rings > 1 && pipe == 1 && queue == 0) {
 177                 int me = ring->me;
 178                 int bit;
 179
 180                 bit = amdgpu_gfx_me_queue_to_bit(adev, me, pipe, queue);
 181                 if (ring == &adev->gfx.gfx_ring[bit])
 182                         return true;
 183         }
 184
 185         return false;
 186 }
 187
 188 bool amdgpu_gfx_is_high_priority_compute_queue(struct amdgpu_device *adev,
 189                                                struct amdgpu_ring *ring)
 190 {
 191         /* Policy: use 1st queue as high priority compute queue if we
 192          * have more than one compute queue.
 193          */
 194         if (adev->gfx.num_compute_rings > 1 &&
 195             ring == &adev->gfx.compute_ring[0])
 196                 return true;
 197
 198         return false;
 199 }
 200
 201 void amdgpu_gfx_compute_queue_acquire(struct amdgpu_device *adev)
 202 {
 203         int i, j, queue, pipe;
 204         bool multipipe_policy = amdgpu_gfx_is_compute_multipipe_capable(adev);
 205         int max_queues_per_mec = min(adev->gfx.mec.num_pipe_per_mec *
 206                                      adev->gfx.mec.num_queue_per_pipe,
 207                                      adev->gfx.num_compute_rings);
 208         int num_xcc = adev->gfx.xcc_mask ? NUM_XCC(adev->gfx.xcc_mask) : 1;
 209
 210         if (multipipe_policy) {
 211                 /* policy: make queues evenly cross all pipes on MEC1 only
 212                  * for multiple xcc, just use the original policy for simplicity */
 213                 for (j = 0; j < num_xcc; j++) {
 214                         for (i = 0; i < max_queues_per_mec; i++) {
 215                                 pipe = i % adev->gfx.mec.num_pipe_per_mec;
 216                                 queue = (i / adev->gfx.mec.num_pipe_per_mec) %
 217                                          adev->gfx.mec.num_queue_per_pipe;
 218
 219                                 set_bit(pipe * adev->gfx.mec.num_queue_per_pipe + queue,
 220                                         adev->gfx.mec_bitmap[j].queue_bitmap);
 221                         }
 222                 }
 223         } else {
 224                 /* policy: amdgpu owns all queues in the given pipe */
 225                 for (j = 0; j < num_xcc; j++) {
 226                         for (i = 0; i < max_queues_per_mec; ++i)
 227                                 set_bit(i, adev->gfx.mec_bitmap[j].queue_bitmap);
 228                 }
 229         }
 230
 231         for (j = 0; j < num_xcc; j++) {
 232                 dev_dbg(adev->dev, "mec queue bitmap weight=%d\n",
 233                         bitmap_weight(adev->gfx.mec_bitmap[j].queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES));
 234         }
 235 }
 236
 237 void amdgpu_gfx_graphics_queue_acquire(struct amdgpu_device *adev)
 238 {
 239         int i, queue, pipe;
 240         bool multipipe_policy = amdgpu_gfx_is_graphics_multipipe_capable(adev);
 241         int max_queues_per_me = adev->gfx.me.num_pipe_per_me *
 242                                         adev->gfx.me.num_queue_per_pipe;
 243
 244         if (multipipe_policy) {
 245                 /* policy: amdgpu owns the first queue per pipe at this stage
 246                  * will extend to mulitple queues per pipe later */
 247                 for (i = 0; i < max_queues_per_me; i++) {
 248                         pipe = i % adev->gfx.me.num_pipe_per_me;
 249                         queue = (i / adev->gfx.me.num_pipe_per_me) %
 250                                 adev->gfx.me.num_queue_per_pipe;
 251
 252                         set_bit(pipe * adev->gfx.me.num_queue_per_pipe + queue,
 253                                 adev->gfx.me.queue_bitmap);
 254                 }
 255         } else {
 256                 for (i = 0; i < max_queues_per_me; ++i)
 257                         set_bit(i, adev->gfx.me.queue_bitmap);
 258         }
 259
 260         /* update the number of active graphics rings */
 261         adev->gfx.num_gfx_rings =
 262                 bitmap_weight(adev->gfx.me.queue_bitmap, AMDGPU_MAX_GFX_QUEUES);
 263 }
 264
 265 static int amdgpu_gfx_kiq_acquire(struct amdgpu_device *adev,
 266                                   struct amdgpu_ring *ring, int xcc_id)
 267 {
 268         int queue_bit;
 269         int mec, pipe, queue;
 270
 271         queue_bit = adev->gfx.mec.num_mec
 272                     * adev->gfx.mec.num_pipe_per_mec
 273                     * adev->gfx.mec.num_queue_per_pipe;
 274
 275         while (--queue_bit >= 0) {
 276                 if (test_bit(queue_bit, adev->gfx.mec_bitmap[xcc_id].queue_bitmap))
 277                         continue;
 278
 279                 amdgpu_queue_mask_bit_to_mec_queue(adev, queue_bit, &mec, &pipe, &queue);
 280
 281                 /*
 282                  * 1. Using pipes 2/3 from MEC 2 seems cause problems.
 283                  * 2. It must use queue id 0, because CGPG_IDLE/SAVE/LOAD/RUN
 284                  * only can be issued on queue 0.
 285                  */
 286                 if ((mec == 1 && pipe > 1) || queue != 0)
 287                         continue;
 288
 289                 ring->me = mec + 1;
 290                 ring->pipe = pipe;
 291                 ring->queue = queue;
 292
 293                 return 0;
 294         }
 295
 296         dev_err(adev->dev, "Failed to find a queue for KIQ\n");
 297         return -EINVAL;
 298 }
 299
 300 int amdgpu_gfx_kiq_init_ring(struct amdgpu_device *adev, int xcc_id)
 301 {
 302         struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];
 303         struct amdgpu_irq_src *irq = &kiq->irq;
 304         struct amdgpu_ring *ring = &kiq->ring;
 305         int r = 0;
 306
 307         spin_lock_init(&kiq->ring_lock);
 308
 309         ring->adev = NULL;
 310         ring->ring_obj = NULL;
 311         ring->use_doorbell = true;
 312         ring->xcc_id = xcc_id;
 313         ring->vm_hub = AMDGPU_GFXHUB(xcc_id);
 314         ring->doorbell_index =
 315                 (adev->doorbell_index.kiq +
 316                  xcc_id * adev->doorbell_index.xcc_doorbell_range)
 317                 << 1;
 318
 319         r = amdgpu_gfx_kiq_acquire(adev, ring, xcc_id);
 320         if (r)
 321                 return r;
 322
 323         ring->eop_gpu_addr = kiq->eop_gpu_addr;
 324         ring->no_scheduler = true;
 325         snprintf(ring->name, sizeof(ring->name), "kiq_%hhu.%hhu.%hhu.%hhu",
 326                  (unsigned char)xcc_id, (unsigned char)ring->me,
 327                  (unsigned char)ring->pipe, (unsigned char)ring->queue);
 328         r = amdgpu_ring_init(adev, ring, 1024, irq, AMDGPU_CP_KIQ_IRQ_DRIVER0,
 329                              AMDGPU_RING_PRIO_DEFAULT, NULL);
 330         if (r)
 331                 dev_warn(adev->dev, "(%d) failed to init kiq ring\n", r);
 332
 333         return r;
 334 }
 335
 336 void amdgpu_gfx_kiq_free_ring(struct amdgpu_ring *ring)
 337 {
 338         amdgpu_ring_fini(ring);
 339 }
 340
 341 void amdgpu_gfx_kiq_fini(struct amdgpu_device *adev, int xcc_id)
 342 {
 343         struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];
 344
 345         amdgpu_bo_free_kernel(&kiq->eop_obj, &kiq->eop_gpu_addr, NULL);
 346 }
 347
 348 int amdgpu_gfx_kiq_init(struct amdgpu_device *adev,
 349                         unsigned int hpd_size, int xcc_id)
 350 {
 351         int r;
 352         u32 *hpd;
 353         struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];
 354
 355         r = amdgpu_bo_create_kernel(adev, hpd_size, PAGE_SIZE,
 356                                     AMDGPU_GEM_DOMAIN_GTT, &kiq->eop_obj,
 357                                     &kiq->eop_gpu_addr, (void **)&hpd);
 358         if (r) {
 359                 dev_warn(adev->dev, "failed to create KIQ bo (%d).\n", r);
 360                 return r;
 361         }
 362
 363         memset(hpd, 0, hpd_size);
 364
 365         r = amdgpu_bo_reserve(kiq->eop_obj, true);
 366         if (unlikely(r != 0))
 367                 dev_warn(adev->dev, "(%d) reserve kiq eop bo failed\n", r);
 368         amdgpu_bo_kunmap(kiq->eop_obj);
 369         amdgpu_bo_unreserve(kiq->eop_obj);
 370
 371         return 0;
 372 }
 373
 374 /* create MQD for each compute/gfx queue */
 375 int amdgpu_gfx_mqd_sw_init(struct amdgpu_device *adev,
 376                            unsigned int mqd_size, int xcc_id)
 377 {
 378         int r, i, j;
 379         struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];
 380         struct amdgpu_ring *ring = &kiq->ring;
 381         u32 domain = AMDGPU_GEM_DOMAIN_GTT;
 382
 383 #if !defined(CONFIG_ARM) && !defined(CONFIG_ARM64)
 384         /* Only enable on gfx10 and 11 for now to avoid changing behavior on older chips */
 385         if (amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(10, 0, 0))
 386                 domain |= AMDGPU_GEM_DOMAIN_VRAM;
 387 #endif
 388
 389         /* create MQD for KIQ */
 390         if (!adev->enable_mes_kiq && !ring->mqd_obj) {
 391                 /* originaly the KIQ MQD is put in GTT domain, but for SRIOV VRAM domain is a must
 392                  * otherwise hypervisor trigger SAVE_VF fail after driver unloaded which mean MQD
 393                  * deallocated and gart_unbind, to strict diverage we decide to use VRAM domain for
 394                  * KIQ MQD no matter SRIOV or Bare-metal
 395                  */
 396                 r = amdgpu_bo_create_kernel(adev, mqd_size, PAGE_SIZE,
 397                                             AMDGPU_GEM_DOMAIN_VRAM |
 398                                             AMDGPU_GEM_DOMAIN_GTT,
 399                                             &ring->mqd_obj,
 400                                             &ring->mqd_gpu_addr,
 401                                             &ring->mqd_ptr);
 402                 if (r) {
 403                         dev_warn(adev->dev, "failed to create ring mqd ob (%d)", r);
 404                         return r;
 405                 }
 406
 407                 /* prepare MQD backup */
 408                 kiq->mqd_backup = kzalloc(mqd_size, GFP_KERNEL);
 409                 if (!kiq->mqd_backup) {
 410                         dev_warn(adev->dev,
 411                                  "no memory to create MQD backup for ring %s\n", ring->name);
 412                         return -ENOMEM;
 413                 }
 414         }
 415
 416         if (adev->asic_type >= CHIP_NAVI10 && amdgpu_async_gfx_ring) {
 417                 /* create MQD for each KGQ */
 418                 for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
 419                         ring = &adev->gfx.gfx_ring[i];
 420                         if (!ring->mqd_obj) {
 421                                 r = amdgpu_bo_create_kernel(adev, mqd_size, PAGE_SIZE,
 422                                                             domain, &ring->mqd_obj,
 423                                                             &ring->mqd_gpu_addr, &ring->mqd_ptr);
 424                                 if (r) {
 425                                         dev_warn(adev->dev, "failed to create ring mqd bo (%d)", r);
 426                                         return r;
 427                                 }
 428
 429                                 ring->mqd_size = mqd_size;
 430                                 /* prepare MQD backup */
 431                                 adev->gfx.me.mqd_backup[i] = kzalloc(mqd_size, GFP_KERNEL);
 432                                 if (!adev->gfx.me.mqd_backup[i]) {
 433                                         dev_warn(adev->dev, "no memory to create MQD backup for ring %s\n", ring->name);
 434                                         return -ENOMEM;
 435                                 }
 436                         }
 437                 }
 438         }
 439
 440         /* create MQD for each KCQ */
 441         for (i = 0; i < adev->gfx.num_compute_rings; i++) {
 442                 j = i + xcc_id * adev->gfx.num_compute_rings;
 443                 ring = &adev->gfx.compute_ring[j];
 444                 if (!ring->mqd_obj) {
 445                         r = amdgpu_bo_create_kernel(adev, mqd_size, PAGE_SIZE,
 446                                                     domain, &ring->mqd_obj,
 447                                                     &ring->mqd_gpu_addr, &ring->mqd_ptr);
 448                         if (r) {
 449                                 dev_warn(adev->dev, "failed to create ring mqd bo (%d)", r);
 450                                 return r;
 451                         }
 452
 453                         ring->mqd_size = mqd_size;
 454                         /* prepare MQD backup */
 455                         adev->gfx.mec.mqd_backup[j] = kzalloc(mqd_size, GFP_KERNEL);
 456                         if (!adev->gfx.mec.mqd_backup[j]) {
 457                                 dev_warn(adev->dev, "no memory to create MQD backup for ring %s\n", ring->name);
 458                                 return -ENOMEM;
 459                         }
 460                 }
 461         }
 462
 463         return 0;
 464 }
 465
 466 void amdgpu_gfx_mqd_sw_fini(struct amdgpu_device *adev, int xcc_id)
 467 {
 468         struct amdgpu_ring *ring = NULL;
 469         int i, j;
 470         struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];
 471
 472         if (adev->asic_type >= CHIP_NAVI10 && amdgpu_async_gfx_ring) {
 473                 for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
 474                         ring = &adev->gfx.gfx_ring[i];
 475                         kfree(adev->gfx.me.mqd_backup[i]);
 476                         amdgpu_bo_free_kernel(&ring->mqd_obj,
 477                                               &ring->mqd_gpu_addr,
 478                                               &ring->mqd_ptr);
 479                 }
 480         }
 481
 482         for (i = 0; i < adev->gfx.num_compute_rings; i++) {
 483                 j = i + xcc_id * adev->gfx.num_compute_rings;
 484                 ring = &adev->gfx.compute_ring[j];
 485                 kfree(adev->gfx.mec.mqd_backup[j]);
 486                 amdgpu_bo_free_kernel(&ring->mqd_obj,
 487                                       &ring->mqd_gpu_addr,
 488                                       &ring->mqd_ptr);
 489         }
 490
 491         ring = &kiq->ring;
 492         kfree(kiq->mqd_backup);
 493         amdgpu_bo_free_kernel(&ring->mqd_obj,
 494                               &ring->mqd_gpu_addr,
 495                               &ring->mqd_ptr);
 496 }
 497
 498 int amdgpu_gfx_disable_kcq(struct amdgpu_device *adev, int xcc_id)
 499 {
 500         struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];
 501         struct amdgpu_ring *kiq_ring = &kiq->ring;
 502         int i, r = 0;
 503         int j;
 504
 505         if (adev->enable_mes) {
 506                 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
 507                         j = i + xcc_id * adev->gfx.num_compute_rings;
 508                         amdgpu_mes_unmap_legacy_queue(adev,
 509                                                    &adev->gfx.compute_ring[j],
 510                                                    RESET_QUEUES, 0, 0);
 511                 }
 512                 return 0;
 513         }
 514
 515         if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues)
 516                 return -EINVAL;
 517
 518         if (!kiq_ring->sched.ready || amdgpu_in_reset(adev))
 519                 return 0;
 520
 521         spin_lock(&kiq->ring_lock);
 522         if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size *
 523                                         adev->gfx.num_compute_rings)) {
 524                 spin_unlock(&kiq->ring_lock);
 525                 return -ENOMEM;
 526         }
 527
 528         for (i = 0; i < adev->gfx.num_compute_rings; i++) {
 529                 j = i + xcc_id * adev->gfx.num_compute_rings;
 530                 kiq->pmf->kiq_unmap_queues(kiq_ring,
 531                                            &adev->gfx.compute_ring[j],
 532                                            RESET_QUEUES, 0, 0);
 533         }
 534         /* Submit unmap queue packet */
 535         amdgpu_ring_commit(kiq_ring);
 536         /*
 537          * Ring test will do a basic scratch register change check. Just run
 538          * this to ensure that unmap queues that is submitted before got
 539          * processed successfully before returning.
 540          */
 541         r = amdgpu_ring_test_helper(kiq_ring);
 542
 543         spin_unlock(&kiq->ring_lock);
 544
 545         return r;
 546 }
 547
 548 int amdgpu_gfx_disable_kgq(struct amdgpu_device *adev, int xcc_id)
 549 {
 550         struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];
 551         struct amdgpu_ring *kiq_ring = &kiq->ring;
 552         int i, r = 0;
 553         int j;
 554
 555         if (adev->enable_mes) {
 556                 if (amdgpu_gfx_is_master_xcc(adev, xcc_id)) {
 557                         for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
 558                                 j = i + xcc_id * adev->gfx.num_gfx_rings;
 559                                 amdgpu_mes_unmap_legacy_queue(adev,
 560                                                       &adev->gfx.gfx_ring[j],
 561                                                       PREEMPT_QUEUES, 0, 0);
 562                         }
 563                 }
 564                 return 0;
 565         }
 566
 567         if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues)
 568                 return -EINVAL;
 569
 570         if (!adev->gfx.kiq[0].ring.sched.ready || amdgpu_in_reset(adev))
 571                 return 0;
 572
 573         if (amdgpu_gfx_is_master_xcc(adev, xcc_id)) {
 574                 spin_lock(&kiq->ring_lock);
 575                 if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size *
 576                                                 adev->gfx.num_gfx_rings)) {
 577                         spin_unlock(&kiq->ring_lock);
 578                         return -ENOMEM;
 579                 }
 580
 581                 for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
 582                         j = i + xcc_id * adev->gfx.num_gfx_rings;
 583                         kiq->pmf->kiq_unmap_queues(kiq_ring,
 584                                                    &adev->gfx.gfx_ring[j],
 585                                                    PREEMPT_QUEUES, 0, 0);
 586                 }
 587                 /* Submit unmap queue packet */
 588                 amdgpu_ring_commit(kiq_ring);
 589
 590                 /*
 591                  * Ring test will do a basic scratch register change check.
 592                  * Just run this to ensure that unmap queues that is submitted
 593                  * before got processed successfully before returning.
 594                  */
 595                 r = amdgpu_ring_test_helper(kiq_ring);
 596                 spin_unlock(&kiq->ring_lock);
 597         }
 598
 599         return r;
 600 }
 601
 602 int amdgpu_queue_mask_bit_to_set_resource_bit(struct amdgpu_device *adev,
 603                                         int queue_bit)
 604 {
 605         int mec, pipe, queue;
 606         int set_resource_bit = 0;
 607
 608         amdgpu_queue_mask_bit_to_mec_queue(adev, queue_bit, &mec, &pipe, &queue);
 609
 610         set_resource_bit = mec * 4 * 8 + pipe * 8 + queue;
 611
 612         return set_resource_bit;
 613 }
 614
 615 static int amdgpu_gfx_mes_enable_kcq(struct amdgpu_device *adev, int xcc_id)
 616 {
 617         struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];
 618         struct amdgpu_ring *kiq_ring = &kiq->ring;
 619         uint64_t queue_mask = ~0ULL;
 620         int r, i, j;
 621
 622         amdgpu_device_flush_hdp(adev, NULL);
 623
 624         if (!adev->enable_uni_mes) {
 625                 spin_lock(&kiq->ring_lock);
 626                 r = amdgpu_ring_alloc(kiq_ring, kiq->pmf->set_resources_size);
 627                 if (r) {
 628                         dev_err(adev->dev, "Failed to lock KIQ (%d).\n", r);
 629                         spin_unlock(&kiq->ring_lock);
 630                         return r;
 631                 }
 632
 633                 kiq->pmf->kiq_set_resources(kiq_ring, queue_mask);
 634                 r = amdgpu_ring_test_helper(kiq_ring);
 635                 spin_unlock(&kiq->ring_lock);
 636                 if (r)
 637                         dev_err(adev->dev, "KIQ failed to set resources\n");
 638         }
 639
 640         for (i = 0; i < adev->gfx.num_compute_rings; i++) {
 641                 j = i + xcc_id * adev->gfx.num_compute_rings;
 642                 r = amdgpu_mes_map_legacy_queue(adev,
 643                                                 &adev->gfx.compute_ring[j]);
 644                 if (r) {
 645                         dev_err(adev->dev, "failed to map compute queue\n");
 646                         return r;
 647                 }
 648         }
 649
 650         return 0;
 651 }
 652
 653 int amdgpu_gfx_enable_kcq(struct amdgpu_device *adev, int xcc_id)
 654 {
 655         struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];
 656         struct amdgpu_ring *kiq_ring = &kiq->ring;
 657         uint64_t queue_mask = 0;
 658         int r, i, j;
 659
 660         if (adev->mes.enable_legacy_queue_map)
 661                 return amdgpu_gfx_mes_enable_kcq(adev, xcc_id);
 662
 663         if (!kiq->pmf || !kiq->pmf->kiq_map_queues || !kiq->pmf->kiq_set_resources)
 664                 return -EINVAL;
 665
 666         for (i = 0; i < AMDGPU_MAX_COMPUTE_QUEUES; ++i) {
 667                 if (!test_bit(i, adev->gfx.mec_bitmap[xcc_id].queue_bitmap))
 668                         continue;
 669
 670                 /* This situation may be hit in the future if a new HW
 671                  * generation exposes more than 64 queues. If so, the
 672                  * definition of queue_mask needs updating */
 673                 if (WARN_ON(i > (sizeof(queue_mask)*8))) {
 674                         DRM_ERROR("Invalid KCQ enabled: %d\n", i);
 675                         break;
 676                 }
 677
 678                 queue_mask |= (1ull << amdgpu_queue_mask_bit_to_set_resource_bit(adev, i));
 679         }
 680
 681         amdgpu_device_flush_hdp(adev, NULL);
 682
 683         DRM_INFO("kiq ring mec %d pipe %d q %d\n", kiq_ring->me, kiq_ring->pipe,
 684                  kiq_ring->queue);
 685
 686         spin_lock(&kiq->ring_lock);
 687         r = amdgpu_ring_alloc(kiq_ring, kiq->pmf->map_queues_size *
 688                                         adev->gfx.num_compute_rings +
 689                                         kiq->pmf->set_resources_size);
 690         if (r) {
 691                 DRM_ERROR("Failed to lock KIQ (%d).\n", r);
 692                 spin_unlock(&kiq->ring_lock);
 693                 return r;
 694         }
 695
 696         kiq->pmf->kiq_set_resources(kiq_ring, queue_mask);
 697         for (i = 0; i < adev->gfx.num_compute_rings; i++) {
 698                 j = i + xcc_id * adev->gfx.num_compute_rings;
 699                 kiq->pmf->kiq_map_queues(kiq_ring,
 700                                          &adev->gfx.compute_ring[j]);
 701         }
 702         /* Submit map queue packet */
 703         amdgpu_ring_commit(kiq_ring);
 704         /*
 705          * Ring test will do a basic scratch register change check. Just run
 706          * this to ensure that map queues that is submitted before got
 707          * processed successfully before returning.
 708          */
 709         r = amdgpu_ring_test_helper(kiq_ring);
 710         spin_unlock(&kiq->ring_lock);
 711         if (r)
 712                 DRM_ERROR("KCQ enable failed\n");
 713
 714         return r;
 715 }
 716
 717 int amdgpu_gfx_enable_kgq(struct amdgpu_device *adev, int xcc_id)
 718 {
 719         struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];
 720         struct amdgpu_ring *kiq_ring = &kiq->ring;
 721         int r, i, j;
 722
 723         if (!kiq->pmf || !kiq->pmf->kiq_map_queues)
 724                 return -EINVAL;
 725
 726         amdgpu_device_flush_hdp(adev, NULL);
 727
 728         if (adev->mes.enable_legacy_queue_map) {
 729                 for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
 730                         j = i + xcc_id * adev->gfx.num_gfx_rings;
 731                         r = amdgpu_mes_map_legacy_queue(adev,
 732                                                         &adev->gfx.gfx_ring[j]);
 733                         if (r) {
 734                                 DRM_ERROR("failed to map gfx queue\n");
 735                                 return r;
 736                         }
 737                 }
 738
 739                 return 0;
 740         }
 741
 742         spin_lock(&kiq->ring_lock);
 743         /* No need to map kcq on the slave */
 744         if (amdgpu_gfx_is_master_xcc(adev, xcc_id)) {
 745                 r = amdgpu_ring_alloc(kiq_ring, kiq->pmf->map_queues_size *
 746                                                 adev->gfx.num_gfx_rings);
 747                 if (r) {
 748                         DRM_ERROR("Failed to lock KIQ (%d).\n", r);
 749                         spin_unlock(&kiq->ring_lock);
 750                         return r;
 751                 }
 752
 753                 for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
 754                         j = i + xcc_id * adev->gfx.num_gfx_rings;
 755                         kiq->pmf->kiq_map_queues(kiq_ring,
 756                                                  &adev->gfx.gfx_ring[j]);
 757                 }
 758         }
 759         /* Submit map queue packet */
 760         amdgpu_ring_commit(kiq_ring);
 761         /*
 762          * Ring test will do a basic scratch register change check. Just run
 763          * this to ensure that map queues that is submitted before got
 764          * processed successfully before returning.
 765          */
 766         r = amdgpu_ring_test_helper(kiq_ring);
 767         spin_unlock(&kiq->ring_lock);
 768         if (r)
 769                 DRM_ERROR("KGQ enable failed\n");
 770
 771         return r;
 772 }
 773
 774 /* amdgpu_gfx_off_ctrl - Handle gfx off feature enable/disable
 775  *
 776  * @adev: amdgpu_device pointer
 777  * @bool enable true: enable gfx off feature, false: disable gfx off feature
 778  *
 779  * 1. gfx off feature will be enabled by gfx ip after gfx cg gp enabled.
 780  * 2. other client can send request to disable gfx off feature, the request should be honored.
 781  * 3. other client can cancel their request of disable gfx off feature
 782  * 4. other client should not send request to enable gfx off feature before disable gfx off feature.
 783  */
 784
 785 void amdgpu_gfx_off_ctrl(struct amdgpu_device *adev, bool enable)
 786 {
 787         unsigned long delay = GFX_OFF_DELAY_ENABLE;
 788
 789         if (!(adev->pm.pp_feature & PP_GFXOFF_MASK))
 790                 return;
 791
 792         mutex_lock(&adev->gfx.gfx_off_mutex);
 793
 794         if (enable) {
 795                 /* If the count is already 0, it means there's an imbalance bug somewhere.
 796                  * Note that the bug may be in a different caller than the one which triggers the
 797                  * WARN_ON_ONCE.
 798                  */
 799                 if (WARN_ON_ONCE(adev->gfx.gfx_off_req_count == 0))
 800                         goto unlock;
 801
 802                 adev->gfx.gfx_off_req_count--;
 803
 804                 if (adev->gfx.gfx_off_req_count == 0 &&
 805                     !adev->gfx.gfx_off_state) {
 806                         /* If going to s2idle, no need to wait */
 807                         if (adev->in_s0ix) {
 808                                 if (!amdgpu_dpm_set_powergating_by_smu(adev,
 809                                                 AMD_IP_BLOCK_TYPE_GFX, true, 0))
 810                                         adev->gfx.gfx_off_state = true;
 811                         } else {
 812                                 schedule_delayed_work(&adev->gfx.gfx_off_delay_work,
 813                                               delay);
 814                         }
 815                 }
 816         } else {
 817                 if (adev->gfx.gfx_off_req_count == 0) {
 818                         cancel_delayed_work_sync(&adev->gfx.gfx_off_delay_work);
 819
 820                         if (adev->gfx.gfx_off_state &&
 821                             !amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, false, 0)) {
 822                                 adev->gfx.gfx_off_state = false;
 823
 824                                 if (adev->gfx.funcs->init_spm_golden) {
 825                                         dev_dbg(adev->dev,
 826                                                 "GFXOFF is disabled, re-init SPM golden settings\n");
 827                                         amdgpu_gfx_init_spm_golden(adev);
 828                                 }
 829                         }
 830                 }
 831
 832                 adev->gfx.gfx_off_req_count++;
 833         }
 834
 835 unlock:
 836         mutex_unlock(&adev->gfx.gfx_off_mutex);
 837 }
 838
 839 int amdgpu_set_gfx_off_residency(struct amdgpu_device *adev, bool value)
 840 {
 841         int r = 0;
 842
 843         mutex_lock(&adev->gfx.gfx_off_mutex);
 844
 845         r = amdgpu_dpm_set_residency_gfxoff(adev, value);
 846
 847         mutex_unlock(&adev->gfx.gfx_off_mutex);
 848
 849         return r;
 850 }
 851
 852 int amdgpu_get_gfx_off_residency(struct amdgpu_device *adev, u32 *value)
 853 {
 854         int r = 0;
 855
 856         mutex_lock(&adev->gfx.gfx_off_mutex);
 857
 858         r = amdgpu_dpm_get_residency_gfxoff(adev, value);
 859
 860         mutex_unlock(&adev->gfx.gfx_off_mutex);
 861
 862         return r;
 863 }
 864
 865 int amdgpu_get_gfx_off_entrycount(struct amdgpu_device *adev, u64 *value)
 866 {
 867         int r = 0;
 868
 869         mutex_lock(&adev->gfx.gfx_off_mutex);
 870
 871         r = amdgpu_dpm_get_entrycount_gfxoff(adev, value);
 872
 873         mutex_unlock(&adev->gfx.gfx_off_mutex);
 874
 875         return r;
 876 }
 877
 878 int amdgpu_get_gfx_off_status(struct amdgpu_device *adev, uint32_t *value)
 879 {
 880
 881         int r = 0;
 882
 883         mutex_lock(&adev->gfx.gfx_off_mutex);
 884
 885         r = amdgpu_dpm_get_status_gfxoff(adev, value);
 886
 887         mutex_unlock(&adev->gfx.gfx_off_mutex);
 888
 889         return r;
 890 }
 891
 892 int amdgpu_gfx_ras_late_init(struct amdgpu_device *adev, struct ras_common_if *ras_block)
 893 {
 894         int r;
 895
 896         if (amdgpu_ras_is_supported(adev, ras_block->block)) {
 897                 if (!amdgpu_persistent_edc_harvesting_supported(adev)) {
 898                         r = amdgpu_ras_reset_error_status(adev, AMDGPU_RAS_BLOCK__GFX);
 899                         if (r)
 900                                 return r;
 901                 }
 902
 903                 r = amdgpu_ras_block_late_init(adev, ras_block);
 904                 if (r)
 905                         return r;
 906
 907                 if (amdgpu_sriov_vf(adev))
 908                         return r;
 909
 910                 if (adev->gfx.cp_ecc_error_irq.funcs) {
 911                         r = amdgpu_irq_get(adev, &adev->gfx.cp_ecc_error_irq, 0);
 912                         if (r)
 913                                 goto late_fini;
 914                 }
 915         } else {
 916                 amdgpu_ras_feature_enable_on_boot(adev, ras_block, 0);
 917         }
 918
 919         return 0;
 920 late_fini:
 921         amdgpu_ras_block_late_fini(adev, ras_block);
 922         return r;
 923 }
 924
 925 int amdgpu_gfx_ras_sw_init(struct amdgpu_device *adev)
 926 {
 927         int err = 0;
 928         struct amdgpu_gfx_ras *ras = NULL;
 929
 930         /* adev->gfx.ras is NULL, which means gfx does not
 931          * support ras function, then do nothing here.
 932          */
 933         if (!adev->gfx.ras)
 934                 return 0;
 935
 936         ras = adev->gfx.ras;
 937
 938         err = amdgpu_ras_register_ras_block(adev, &ras->ras_block);
 939         if (err) {
 940                 dev_err(adev->dev, "Failed to register gfx ras block!\n");
 941                 return err;
 942         }
 943
 944         strcpy(ras->ras_block.ras_comm.name, "gfx");
 945         ras->ras_block.ras_comm.block = AMDGPU_RAS_BLOCK__GFX;
 946         ras->ras_block.ras_comm.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE;
 947         adev->gfx.ras_if = &ras->ras_block.ras_comm;
 948
 949         /* If not define special ras_late_init function, use gfx default ras_late_init */
 950         if (!ras->ras_block.ras_late_init)
 951                 ras->ras_block.ras_late_init = amdgpu_gfx_ras_late_init;
 952
 953         /* If not defined special ras_cb function, use default ras_cb */
 954         if (!ras->ras_block.ras_cb)
 955                 ras->ras_block.ras_cb = amdgpu_gfx_process_ras_data_cb;
 956
 957         return 0;
 958 }
 959
 960 int amdgpu_gfx_poison_consumption_handler(struct amdgpu_device *adev,
 961                                                 struct amdgpu_iv_entry *entry)
 962 {
 963         if (adev->gfx.ras && adev->gfx.ras->poison_consumption_handler)
 964                 return adev->gfx.ras->poison_consumption_handler(adev, entry);
 965
 966         return 0;
 967 }
 968
 969 int amdgpu_gfx_process_ras_data_cb(struct amdgpu_device *adev,
 970                 void *err_data,
 971                 struct amdgpu_iv_entry *entry)
 972 {
 973         /* TODO ue will trigger an interrupt.
 974          *
 975          * When “Full RAS” is enabled, the per-IP interrupt sources should
 976          * be disabled and the driver should only look for the aggregated
 977          * interrupt via sync flood
 978          */
 979         if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) {
 980                 kgd2kfd_set_sram_ecc_flag(adev->kfd.dev);
 981                 if (adev->gfx.ras && adev->gfx.ras->ras_block.hw_ops &&
 982                     adev->gfx.ras->ras_block.hw_ops->query_ras_error_count)
 983                         adev->gfx.ras->ras_block.hw_ops->query_ras_error_count(adev, err_data);
 984                 amdgpu_ras_reset_gpu(adev);
 985         }
 986         return AMDGPU_RAS_SUCCESS;
 987 }
 988
 989 int amdgpu_gfx_cp_ecc_error_irq(struct amdgpu_device *adev,
 990                                   struct amdgpu_irq_src *source,
 991                                   struct amdgpu_iv_entry *entry)
 992 {
 993         struct ras_common_if *ras_if = adev->gfx.ras_if;
 994         struct ras_dispatch_if ih_data = {
 995                 .entry = entry,
 996         };
 997
 998         if (!ras_if)
 999                 return 0;
1000
1001         ih_data.head = *ras_if;
1002
1003         DRM_ERROR("CP ECC ERROR IRQ\n");
1004         amdgpu_ras_interrupt_dispatch(adev, &ih_data);
1005         return 0;
1006 }
1007
1008 void amdgpu_gfx_ras_error_func(struct amdgpu_device *adev,
1009                 void *ras_error_status,
1010                 void (*func)(struct amdgpu_device *adev, void *ras_error_status,
1011                                 int xcc_id))
1012 {
1013         int i;
1014         int num_xcc = adev->gfx.xcc_mask ? NUM_XCC(adev->gfx.xcc_mask) : 1;
1015         uint32_t xcc_mask = GENMASK(num_xcc - 1, 0);
1016         struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status;
1017
1018         if (err_data) {
1019                 err_data->ue_count = 0;
1020                 err_data->ce_count = 0;
1021         }
1022
1023         for_each_inst(i, xcc_mask)
1024                 func(adev, ras_error_status, i);
1025 }
1026
1027 uint32_t amdgpu_kiq_rreg(struct amdgpu_device *adev, uint32_t reg, uint32_t xcc_id)
1028 {
1029         signed long r, cnt = 0;
1030         unsigned long flags;
1031         uint32_t seq, reg_val_offs = 0, value = 0;
1032         struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];
1033         struct amdgpu_ring *ring = &kiq->ring;
1034
1035         if (amdgpu_device_skip_hw_access(adev))
1036                 return 0;
1037
1038         if (adev->mes.ring[0].sched.ready)
1039                 return amdgpu_mes_rreg(adev, reg);
1040
1041         BUG_ON(!ring->funcs->emit_rreg);
1042
1043         spin_lock_irqsave(&kiq->ring_lock, flags);
1044         if (amdgpu_device_wb_get(adev, &reg_val_offs)) {
1045                 pr_err("critical bug! too many kiq readers\n");
1046                 goto failed_unlock;
1047         }
1048         r = amdgpu_ring_alloc(ring, 32);
1049         if (r)
1050                 goto failed_unlock;
1051
1052         amdgpu_ring_emit_rreg(ring, reg, reg_val_offs);
1053         r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT);
1054         if (r)
1055                 goto failed_undo;
1056
1057         amdgpu_ring_commit(ring);
1058         spin_unlock_irqrestore(&kiq->ring_lock, flags);
1059
1060         r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
1061
1062         /* don't wait anymore for gpu reset case because this way may
1063          * block gpu_recover() routine forever, e.g. this virt_kiq_rreg
1064          * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will
1065          * never return if we keep waiting in virt_kiq_rreg, which cause
1066          * gpu_recover() hang there.
1067          *
1068          * also don't wait anymore for IRQ context
1069          * */
1070         if (r < 1 && (amdgpu_in_reset(adev) || in_interrupt()))
1071                 goto failed_kiq_read;
1072
1073         might_sleep();
1074         while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) {
1075                 msleep(MAX_KIQ_REG_BAILOUT_INTERVAL);
1076                 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
1077         }
1078
1079         if (cnt > MAX_KIQ_REG_TRY)
1080                 goto failed_kiq_read;
1081
1082         mb();
1083         value = adev->wb.wb[reg_val_offs];
1084         amdgpu_device_wb_free(adev, reg_val_offs);
1085         return value;
1086
1087 failed_undo:
1088         amdgpu_ring_undo(ring);
1089 failed_unlock:
1090         spin_unlock_irqrestore(&kiq->ring_lock, flags);
1091 failed_kiq_read:
1092         if (reg_val_offs)
1093                 amdgpu_device_wb_free(adev, reg_val_offs);
1094         dev_err(adev->dev, "failed to read reg:%x\n", reg);
1095         return ~0;
1096 }
1097
1098 void amdgpu_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v, uint32_t xcc_id)
1099 {
1100         signed long r, cnt = 0;
1101         unsigned long flags;
1102         uint32_t seq;
1103         struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];
1104         struct amdgpu_ring *ring = &kiq->ring;
1105
1106         BUG_ON(!ring->funcs->emit_wreg);
1107
1108         if (amdgpu_device_skip_hw_access(adev))
1109                 return;
1110
1111         if (adev->mes.ring[0].sched.ready) {
1112                 amdgpu_mes_wreg(adev, reg, v);
1113                 return;
1114         }
1115
1116         spin_lock_irqsave(&kiq->ring_lock, flags);
1117         r = amdgpu_ring_alloc(ring, 32);
1118         if (r)
1119                 goto failed_unlock;
1120
1121         amdgpu_ring_emit_wreg(ring, reg, v);
1122         r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT);
1123         if (r)
1124                 goto failed_undo;
1125
1126         amdgpu_ring_commit(ring);
1127         spin_unlock_irqrestore(&kiq->ring_lock, flags);
1128
1129         r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
1130
1131         /* don't wait anymore for gpu reset case because this way may
1132          * block gpu_recover() routine forever, e.g. this virt_kiq_rreg
1133          * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will
1134          * never return if we keep waiting in virt_kiq_rreg, which cause
1135          * gpu_recover() hang there.
1136          *
1137          * also don't wait anymore for IRQ context
1138          * */
1139         if (r < 1 && (amdgpu_in_reset(adev) || in_interrupt()))
1140                 goto failed_kiq_write;
1141
1142         might_sleep();
1143         while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) {
1144
1145                 msleep(MAX_KIQ_REG_BAILOUT_INTERVAL);
1146                 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
1147         }
1148
1149         if (cnt > MAX_KIQ_REG_TRY)
1150                 goto failed_kiq_write;
1151
1152         return;
1153
1154 failed_undo:
1155         amdgpu_ring_undo(ring);
1156 failed_unlock:
1157         spin_unlock_irqrestore(&kiq->ring_lock, flags);
1158 failed_kiq_write:
1159         dev_err(adev->dev, "failed to write reg:%x\n", reg);
1160 }
1161
1162 int amdgpu_gfx_get_num_kcq(struct amdgpu_device *adev)
1163 {
1164         if (amdgpu_num_kcq == -1) {
1165                 return 8;
1166         } else if (amdgpu_num_kcq > 8 || amdgpu_num_kcq < 0) {
1167                 dev_warn(adev->dev, "set kernel compute queue number to 8 due to invalid parameter provided by user\n");
1168                 return 8;
1169         }
1170         return amdgpu_num_kcq;
1171 }
1172
1173 void amdgpu_gfx_cp_init_microcode(struct amdgpu_device *adev,
1174                                   uint32_t ucode_id)
1175 {
1176         const struct gfx_firmware_header_v1_0 *cp_hdr;
1177         const struct gfx_firmware_header_v2_0 *cp_hdr_v2_0;
1178         struct amdgpu_firmware_info *info = NULL;
1179         const struct firmware *ucode_fw;
1180         unsigned int fw_size;
1181
1182         switch (ucode_id) {
1183         case AMDGPU_UCODE_ID_CP_PFP:
1184                 cp_hdr = (const struct gfx_firmware_header_v1_0 *)
1185                         adev->gfx.pfp_fw->data;
1186                 adev->gfx.pfp_fw_version =
1187                         le32_to_cpu(cp_hdr->header.ucode_version);
1188                 adev->gfx.pfp_feature_version =
1189                         le32_to_cpu(cp_hdr->ucode_feature_version);
1190                 ucode_fw = adev->gfx.pfp_fw;
1191                 fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes);
1192                 break;
1193         case AMDGPU_UCODE_ID_CP_RS64_PFP:
1194                 cp_hdr_v2_0 = (const struct gfx_firmware_header_v2_0 *)
1195                         adev->gfx.pfp_fw->data;
1196                 adev->gfx.pfp_fw_version =
1197                         le32_to_cpu(cp_hdr_v2_0->header.ucode_version);
1198                 adev->gfx.pfp_feature_version =
1199                         le32_to_cpu(cp_hdr_v2_0->ucode_feature_version);
1200                 ucode_fw = adev->gfx.pfp_fw;
1201                 fw_size = le32_to_cpu(cp_hdr_v2_0->ucode_size_bytes);
1202                 break;
1203         case AMDGPU_UCODE_ID_CP_RS64_PFP_P0_STACK:
1204         case AMDGPU_UCODE_ID_CP_RS64_PFP_P1_STACK:
1205                 cp_hdr_v2_0 = (const struct gfx_firmware_header_v2_0 *)
1206                         adev->gfx.pfp_fw->data;
1207                 ucode_fw = adev->gfx.pfp_fw;
1208                 fw_size = le32_to_cpu(cp_hdr_v2_0->data_size_bytes);
1209                 break;
1210         case AMDGPU_UCODE_ID_CP_ME:
1211                 cp_hdr = (const struct gfx_firmware_header_v1_0 *)
1212                         adev->gfx.me_fw->data;
1213                 adev->gfx.me_fw_version =
1214                         le32_to_cpu(cp_hdr->header.ucode_version);
1215                 adev->gfx.me_feature_version =
1216                         le32_to_cpu(cp_hdr->ucode_feature_version);
1217                 ucode_fw = adev->gfx.me_fw;
1218                 fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes);
1219                 break;
1220         case AMDGPU_UCODE_ID_CP_RS64_ME:
1221                 cp_hdr_v2_0 = (const struct gfx_firmware_header_v2_0 *)
1222                         adev->gfx.me_fw->data;
1223                 adev->gfx.me_fw_version =
1224                         le32_to_cpu(cp_hdr_v2_0->header.ucode_version);
1225                 adev->gfx.me_feature_version =
1226                         le32_to_cpu(cp_hdr_v2_0->ucode_feature_version);
1227                 ucode_fw = adev->gfx.me_fw;
1228                 fw_size = le32_to_cpu(cp_hdr_v2_0->ucode_size_bytes);
1229                 break;
1230         case AMDGPU_UCODE_ID_CP_RS64_ME_P0_STACK:
1231         case AMDGPU_UCODE_ID_CP_RS64_ME_P1_STACK:
1232                 cp_hdr_v2_0 = (const struct gfx_firmware_header_v2_0 *)
1233                         adev->gfx.me_fw->data;
1234                 ucode_fw = adev->gfx.me_fw;
1235                 fw_size = le32_to_cpu(cp_hdr_v2_0->data_size_bytes);
1236                 break;
1237         case AMDGPU_UCODE_ID_CP_CE:
1238                 cp_hdr = (const struct gfx_firmware_header_v1_0 *)
1239                         adev->gfx.ce_fw->data;
1240                 adev->gfx.ce_fw_version =
1241                         le32_to_cpu(cp_hdr->header.ucode_version);
1242                 adev->gfx.ce_feature_version =
1243                         le32_to_cpu(cp_hdr->ucode_feature_version);
1244                 ucode_fw = adev->gfx.ce_fw;
1245                 fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes);
1246                 break;
1247         case AMDGPU_UCODE_ID_CP_MEC1:
1248                 cp_hdr = (const struct gfx_firmware_header_v1_0 *)
1249                         adev->gfx.mec_fw->data;
1250                 adev->gfx.mec_fw_version =
1251                         le32_to_cpu(cp_hdr->header.ucode_version);
1252                 adev->gfx.mec_feature_version =
1253                         le32_to_cpu(cp_hdr->ucode_feature_version);
1254                 ucode_fw = adev->gfx.mec_fw;
1255                 fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes) -
1256                           le32_to_cpu(cp_hdr->jt_size) * 4;
1257                 break;
1258         case AMDGPU_UCODE_ID_CP_MEC1_JT:
1259                 cp_hdr = (const struct gfx_firmware_header_v1_0 *)
1260                         adev->gfx.mec_fw->data;
1261                 ucode_fw = adev->gfx.mec_fw;
1262                 fw_size = le32_to_cpu(cp_hdr->jt_size) * 4;
1263                 break;
1264         case AMDGPU_UCODE_ID_CP_MEC2:
1265                 cp_hdr = (const struct gfx_firmware_header_v1_0 *)
1266                         adev->gfx.mec2_fw->data;
1267                 adev->gfx.mec2_fw_version =
1268                         le32_to_cpu(cp_hdr->header.ucode_version);
1269                 adev->gfx.mec2_feature_version =
1270                         le32_to_cpu(cp_hdr->ucode_feature_version);
1271                 ucode_fw = adev->gfx.mec2_fw;
1272                 fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes) -
1273                           le32_to_cpu(cp_hdr->jt_size) * 4;
1274                 break;
1275         case AMDGPU_UCODE_ID_CP_MEC2_JT:
1276                 cp_hdr = (const struct gfx_firmware_header_v1_0 *)
1277                         adev->gfx.mec2_fw->data;
1278                 ucode_fw = adev->gfx.mec2_fw;
1279                 fw_size = le32_to_cpu(cp_hdr->jt_size) * 4;
1280                 break;
1281         case AMDGPU_UCODE_ID_CP_RS64_MEC:
1282                 cp_hdr_v2_0 = (const struct gfx_firmware_header_v2_0 *)
1283                         adev->gfx.mec_fw->data;
1284                 adev->gfx.mec_fw_version =
1285                         le32_to_cpu(cp_hdr_v2_0->header.ucode_version);
1286                 adev->gfx.mec_feature_version =
1287                         le32_to_cpu(cp_hdr_v2_0->ucode_feature_version);
1288                 ucode_fw = adev->gfx.mec_fw;
1289                 fw_size = le32_to_cpu(cp_hdr_v2_0->ucode_size_bytes);
1290                 break;
1291         case AMDGPU_UCODE_ID_CP_RS64_MEC_P0_STACK:
1292         case AMDGPU_UCODE_ID_CP_RS64_MEC_P1_STACK:
1293         case AMDGPU_UCODE_ID_CP_RS64_MEC_P2_STACK:
1294         case AMDGPU_UCODE_ID_CP_RS64_MEC_P3_STACK:
1295                 cp_hdr_v2_0 = (const struct gfx_firmware_header_v2_0 *)
1296                         adev->gfx.mec_fw->data;
1297                 ucode_fw = adev->gfx.mec_fw;
1298                 fw_size = le32_to_cpu(cp_hdr_v2_0->data_size_bytes);
1299                 break;
1300         default:
1301                 dev_err(adev->dev, "Invalid ucode id %u\n", ucode_id);
1302                 return;
1303         }
1304
1305         if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
1306                 info = &adev->firmware.ucode[ucode_id];
1307                 info->ucode_id = ucode_id;
1308                 info->fw = ucode_fw;
1309                 adev->firmware.fw_size += ALIGN(fw_size, PAGE_SIZE);
1310         }
1311 }
1312
1313 bool amdgpu_gfx_is_master_xcc(struct amdgpu_device *adev, int xcc_id)
1314 {
1315         return !(xcc_id % (adev->gfx.num_xcc_per_xcp ?
1316                         adev->gfx.num_xcc_per_xcp : 1));
1317 }
1318
1319 static ssize_t amdgpu_gfx_get_current_compute_partition(struct device *dev,
1320                                                 struct device_attribute *addr,
1321                                                 char *buf)
1322 {
1323         struct drm_device *ddev = dev_get_drvdata(dev);
1324         struct amdgpu_device *adev = drm_to_adev(ddev);
1325         int mode;
1326
1327         mode = amdgpu_xcp_query_partition_mode(adev->xcp_mgr,
1328                                                AMDGPU_XCP_FL_NONE);
1329
1330         return sysfs_emit(buf, "%s\n", amdgpu_gfx_compute_mode_desc(mode));
1331 }
1332
1333 static ssize_t amdgpu_gfx_set_compute_partition(struct device *dev,
1334                                                 struct device_attribute *addr,
1335                                                 const char *buf, size_t count)
1336 {
1337         struct drm_device *ddev = dev_get_drvdata(dev);
1338         struct amdgpu_device *adev = drm_to_adev(ddev);
1339         enum amdgpu_gfx_partition mode;
1340         int ret = 0, num_xcc;
1341
1342         num_xcc = NUM_XCC(adev->gfx.xcc_mask);
1343         if (num_xcc % 2 != 0)
1344                 return -EINVAL;
1345
1346         if (!strncasecmp("SPX", buf, strlen("SPX"))) {
1347                 mode = AMDGPU_SPX_PARTITION_MODE;
1348         } else if (!strncasecmp("DPX", buf, strlen("DPX"))) {
1349                 /*
1350                  * DPX mode needs AIDs to be in multiple of 2.
1351                  * Each AID connects 2 XCCs.
1352                  */
1353                 if (num_xcc%4)
1354                         return -EINVAL;
1355                 mode = AMDGPU_DPX_PARTITION_MODE;
1356         } else if (!strncasecmp("TPX", buf, strlen("TPX"))) {
1357                 if (num_xcc != 6)
1358                         return -EINVAL;
1359                 mode = AMDGPU_TPX_PARTITION_MODE;
1360         } else if (!strncasecmp("QPX", buf, strlen("QPX"))) {
1361                 if (num_xcc != 8)
1362                         return -EINVAL;
1363                 mode = AMDGPU_QPX_PARTITION_MODE;
1364         } else if (!strncasecmp("CPX", buf, strlen("CPX"))) {
1365                 mode = AMDGPU_CPX_PARTITION_MODE;
1366         } else {
1367                 return -EINVAL;
1368         }
1369
1370         ret = amdgpu_xcp_switch_partition_mode(adev->xcp_mgr, mode);
1371
1372         if (ret)
1373                 return ret;
1374
1375         return count;
1376 }
1377
1378 static const char *xcp_desc[] = {
1379         [AMDGPU_SPX_PARTITION_MODE] = "SPX",
1380         [AMDGPU_DPX_PARTITION_MODE] = "DPX",
1381         [AMDGPU_TPX_PARTITION_MODE] = "TPX",
1382         [AMDGPU_QPX_PARTITION_MODE] = "QPX",
1383         [AMDGPU_CPX_PARTITION_MODE] = "CPX",
1384 };
1385
1386 static ssize_t amdgpu_gfx_get_available_compute_partition(struct device *dev,
1387                                                 struct device_attribute *addr,
1388                                                 char *buf)
1389 {
1390         struct drm_device *ddev = dev_get_drvdata(dev);
1391         struct amdgpu_device *adev = drm_to_adev(ddev);
1392         struct amdgpu_xcp_mgr *xcp_mgr = adev->xcp_mgr;
1393         int size = 0, mode;
1394         char *sep = "";
1395
1396         if (!xcp_mgr || !xcp_mgr->avail_xcp_modes)
1397                 return sysfs_emit(buf, "Not supported\n");
1398
1399         for_each_inst(mode, xcp_mgr->avail_xcp_modes) {
1400                 size += sysfs_emit_at(buf, size, "%s%s", sep, xcp_desc[mode]);
1401                 sep = ", ";
1402         }
1403
1404         size += sysfs_emit_at(buf, size, "\n");
1405
1406         return size;
1407 }
1408
1409 static int amdgpu_gfx_run_cleaner_shader_job(struct amdgpu_ring *ring)
1410 {
1411         struct amdgpu_device *adev = ring->adev;
1412         struct drm_gpu_scheduler *sched = &ring->sched;
1413         struct drm_sched_entity entity;
1414         struct dma_fence *f;
1415         struct amdgpu_job *job;
1416         struct amdgpu_ib *ib;
1417         int i, r;
1418
1419         /* Initialize the scheduler entity */
1420         r = drm_sched_entity_init(&entity, DRM_SCHED_PRIORITY_NORMAL,
1421                                   &sched, 1, NULL);
1422         if (r) {
1423                 dev_err(adev->dev, "Failed setting up GFX kernel entity.\n");
1424                 goto err;
1425         }
1426
1427         r = amdgpu_job_alloc_with_ib(ring->adev, &entity, NULL,
1428                                      64, 0,
1429                                      &job);
1430         if (r)
1431                 goto err;
1432
1433         job->enforce_isolation = true;
1434
1435         ib = &job->ibs[0];
1436         for (i = 0; i <= ring->funcs->align_mask; ++i)
1437                 ib->ptr[i] = ring->funcs->nop;
1438         ib->length_dw = ring->funcs->align_mask + 1;
1439
1440         f = amdgpu_job_submit(job);
1441
1442         r = dma_fence_wait(f, false);
1443         if (r)
1444                 goto err;
1445
1446         dma_fence_put(f);
1447
1448         /* Clean up the scheduler entity */
1449         drm_sched_entity_destroy(&entity);
1450         return 0;
1451
1452 err:
1453         return r;
1454 }
1455
1456 static int amdgpu_gfx_run_cleaner_shader(struct amdgpu_device *adev, int xcp_id)
1457 {
1458         int num_xcc = NUM_XCC(adev->gfx.xcc_mask);
1459         struct amdgpu_ring *ring;
1460         int num_xcc_to_clear;
1461         int i, r, xcc_id;
1462
1463         if (adev->gfx.num_xcc_per_xcp)
1464                 num_xcc_to_clear = adev->gfx.num_xcc_per_xcp;
1465         else
1466                 num_xcc_to_clear = 1;
1467
1468         for (xcc_id = 0; xcc_id < num_xcc; xcc_id++) {
1469                 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
1470                         ring = &adev->gfx.compute_ring[i + xcc_id * adev->gfx.num_compute_rings];
1471                         if ((ring->xcp_id == xcp_id) && ring->sched.ready) {
1472                                 r = amdgpu_gfx_run_cleaner_shader_job(ring);
1473                                 if (r)
1474                                         return r;
1475                                 num_xcc_to_clear--;
1476                                 break;
1477                         }
1478                 }
1479         }
1480
1481         if (num_xcc_to_clear)
1482                 return -ENOENT;
1483
1484         return 0;
1485 }
1486
1487 /**
1488  * amdgpu_gfx_set_run_cleaner_shader - Execute the AMDGPU GFX Cleaner Shader
1489  * @dev: The device structure
1490  * @attr: The device attribute structure
1491  * @buf: The buffer containing the input data
1492  * @count: The size of the input data
1493  *
1494  * Provides the sysfs interface to manually run a cleaner shader, which is
1495  * used to clear the GPU state between different tasks. Writing a value to the
1496  * 'run_cleaner_shader' sysfs file triggers the cleaner shader execution.
1497  * The value written corresponds to the partition index on multi-partition
1498  * devices. On single-partition devices, the value should be '0'.
1499  *
1500  * The cleaner shader clears the Local Data Store (LDS) and General Purpose
1501  * Registers (GPRs) to ensure data isolation between GPU workloads.
1502  *
1503  * Return: The number of bytes written to the sysfs file.
1504  */
1505 static ssize_t amdgpu_gfx_set_run_cleaner_shader(struct device *dev,
1506                                                  struct device_attribute *attr,
1507                                                  const char *buf,
1508                                                  size_t count)
1509 {
1510         struct drm_device *ddev = dev_get_drvdata(dev);
1511         struct amdgpu_device *adev = drm_to_adev(ddev);
1512         int ret;
1513         long value;
1514
1515         if (amdgpu_in_reset(adev))
1516                 return -EPERM;
1517         if (adev->in_suspend && !adev->in_runpm)
1518                 return -EPERM;
1519
1520         ret = kstrtol(buf, 0, &value);
1521
1522         if (ret)
1523                 return -EINVAL;
1524
1525         if (value < 0)
1526                 return -EINVAL;
1527
1528         if (adev->xcp_mgr) {
1529                 if (value >= adev->xcp_mgr->num_xcps)
1530                         return -EINVAL;
1531         } else {
1532                 if (value > 1)
1533                         return -EINVAL;
1534         }
1535
1536         ret = pm_runtime_get_sync(ddev->dev);
1537         if (ret < 0) {
1538                 pm_runtime_put_autosuspend(ddev->dev);
1539                 return ret;
1540         }
1541
1542         ret = amdgpu_gfx_run_cleaner_shader(adev, value);
1543
1544         pm_runtime_mark_last_busy(ddev->dev);
1545         pm_runtime_put_autosuspend(ddev->dev);
1546
1547         if (ret)
1548                 return ret;
1549
1550         return count;
1551 }
1552
1553 /**
1554  * amdgpu_gfx_get_enforce_isolation - Query AMDGPU GFX Enforce Isolation Settings
1555  * @dev: The device structure
1556  * @attr: The device attribute structure
1557  * @buf: The buffer to store the output data
1558  *
1559  * Provides the sysfs read interface to get the current settings of the 'enforce_isolation'
1560  * feature for each GPU partition. Reading from the 'enforce_isolation'
1561  * sysfs file returns the isolation settings for all partitions, where '0'
1562  * indicates disabled and '1' indicates enabled.
1563  *
1564  * Return: The number of bytes read from the sysfs file.
1565  */
1566 static ssize_t amdgpu_gfx_get_enforce_isolation(struct device *dev,
1567                                                 struct device_attribute *attr,
1568                                                 char *buf)
1569 {
1570         struct drm_device *ddev = dev_get_drvdata(dev);
1571         struct amdgpu_device *adev = drm_to_adev(ddev);
1572         int i;
1573         ssize_t size = 0;
1574
1575         if (adev->xcp_mgr) {
1576                 for (i = 0; i < adev->xcp_mgr->num_xcps; i++) {
1577                         size += sysfs_emit_at(buf, size, "%u", adev->enforce_isolation[i]);
1578                         if (i < (adev->xcp_mgr->num_xcps - 1))
1579                                 size += sysfs_emit_at(buf, size, " ");
1580                 }
1581                 buf[size++] = '\n';
1582         } else {
1583                 size = sysfs_emit_at(buf, 0, "%u\n", adev->enforce_isolation[0]);
1584         }
1585
1586         return size;
1587 }
1588
1589 /**
1590  * amdgpu_gfx_set_enforce_isolation - Control AMDGPU GFX Enforce Isolation
1591  * @dev: The device structure
1592  * @attr: The device attribute structure
1593  * @buf: The buffer containing the input data
1594  * @count: The size of the input data
1595  *
1596  * This function allows control over the 'enforce_isolation' feature, which
1597  * serializes access to the graphics engine. Writing '1' or '0' to the
1598  * 'enforce_isolation' sysfs file enables or disables process isolation for
1599  * each partition. The input should specify the setting for all partitions.
1600  *
1601  * Return: The number of bytes written to the sysfs file.
1602  */
1603 static ssize_t amdgpu_gfx_set_enforce_isolation(struct device *dev,
1604                                                 struct device_attribute *attr,
1605                                                 const char *buf, size_t count)
1606 {
1607         struct drm_device *ddev = dev_get_drvdata(dev);
1608         struct amdgpu_device *adev = drm_to_adev(ddev);
1609         long partition_values[MAX_XCP] = {0};
1610         int ret, i, num_partitions;
1611         const char *input_buf = buf;
1612
1613         for (i = 0; i < (adev->xcp_mgr ? adev->xcp_mgr->num_xcps : 1); i++) {
1614                 ret = sscanf(input_buf, "%ld", &partition_values[i]);
1615                 if (ret <= 0)
1616                         break;
1617
1618                 /* Move the pointer to the next value in the string */
1619                 input_buf = strchr(input_buf, ' ');
1620                 if (input_buf) {
1621                         input_buf++;
1622                 } else {
1623                         i++;
1624                         break;
1625                 }
1626         }
1627         num_partitions = i;
1628
1629         if (adev->xcp_mgr && num_partitions != adev->xcp_mgr->num_xcps)
1630                 return -EINVAL;
1631
1632         if (!adev->xcp_mgr && num_partitions != 1)
1633                 return -EINVAL;
1634
1635         for (i = 0; i < num_partitions; i++) {
1636                 if (partition_values[i] != 0 && partition_values[i] != 1)
1637                         return -EINVAL;
1638         }
1639
1640         mutex_lock(&adev->enforce_isolation_mutex);
1641
1642         for (i = 0; i < num_partitions; i++) {
1643                 if (adev->enforce_isolation[i] && !partition_values[i]) {
1644                         /* Going from enabled to disabled */
1645                         amdgpu_vmid_free_reserved(adev, AMDGPU_GFXHUB(i));
1646                         amdgpu_mes_set_enforce_isolation(adev, i, false);
1647                 } else if (!adev->enforce_isolation[i] && partition_values[i]) {
1648                         /* Going from disabled to enabled */
1649                         amdgpu_vmid_alloc_reserved(adev, AMDGPU_GFXHUB(i));
1650                         amdgpu_mes_set_enforce_isolation(adev, i, true);
1651                 }
1652                 adev->enforce_isolation[i] = partition_values[i];
1653         }
1654
1655         mutex_unlock(&adev->enforce_isolation_mutex);
1656
1657         return count;
1658 }
1659
1660 static ssize_t amdgpu_gfx_get_gfx_reset_mask(struct device *dev,
1661                                                 struct device_attribute *attr,
1662                                                 char *buf)
1663 {
1664         struct drm_device *ddev = dev_get_drvdata(dev);
1665         struct amdgpu_device *adev = drm_to_adev(ddev);
1666
1667         if (!adev)
1668                 return -ENODEV;
1669
1670         return amdgpu_show_reset_mask(buf, adev->gfx.gfx_supported_reset);
1671 }
1672
1673 static ssize_t amdgpu_gfx_get_compute_reset_mask(struct device *dev,
1674                                                 struct device_attribute *attr,
1675                                                 char *buf)
1676 {
1677         struct drm_device *ddev = dev_get_drvdata(dev);
1678         struct amdgpu_device *adev = drm_to_adev(ddev);
1679
1680         if (!adev)
1681                 return -ENODEV;
1682
1683         return amdgpu_show_reset_mask(buf, adev->gfx.compute_supported_reset);
1684 }
1685
1686 static DEVICE_ATTR(run_cleaner_shader, 0200,
1687                    NULL, amdgpu_gfx_set_run_cleaner_shader);
1688
1689 static DEVICE_ATTR(enforce_isolation, 0644,
1690                    amdgpu_gfx_get_enforce_isolation,
1691                    amdgpu_gfx_set_enforce_isolation);
1692
1693 static DEVICE_ATTR(current_compute_partition, 0644,
1694                    amdgpu_gfx_get_current_compute_partition,
1695                    amdgpu_gfx_set_compute_partition);
1696
1697 static DEVICE_ATTR(available_compute_partition, 0444,
1698                    amdgpu_gfx_get_available_compute_partition, NULL);
1699 static DEVICE_ATTR(gfx_reset_mask, 0444,
1700                    amdgpu_gfx_get_gfx_reset_mask, NULL);
1701
1702 static DEVICE_ATTR(compute_reset_mask, 0444,
1703                    amdgpu_gfx_get_compute_reset_mask, NULL);
1704
1705 static int amdgpu_gfx_sysfs_xcp_init(struct amdgpu_device *adev)
1706 {
1707         struct amdgpu_xcp_mgr *xcp_mgr = adev->xcp_mgr;
1708         bool xcp_switch_supported;
1709         int r;
1710
1711         if (!xcp_mgr)
1712                 return 0;
1713
1714         xcp_switch_supported =
1715                 (xcp_mgr->funcs && xcp_mgr->funcs->switch_partition_mode);
1716
1717         if (!xcp_switch_supported)
1718                 dev_attr_current_compute_partition.attr.mode &=
1719                         ~(S_IWUSR | S_IWGRP | S_IWOTH);
1720
1721         r = device_create_file(adev->dev, &dev_attr_current_compute_partition);
1722         if (r)
1723                 return r;
1724
1725         if (xcp_switch_supported)
1726                 r = device_create_file(adev->dev,
1727                                        &dev_attr_available_compute_partition);
1728
1729         return r;
1730 }
1731
1732 static void amdgpu_gfx_sysfs_xcp_fini(struct amdgpu_device *adev)
1733 {
1734         struct amdgpu_xcp_mgr *xcp_mgr = adev->xcp_mgr;
1735         bool xcp_switch_supported;
1736
1737         if (!xcp_mgr)
1738                 return;
1739
1740         xcp_switch_supported =
1741                 (xcp_mgr->funcs && xcp_mgr->funcs->switch_partition_mode);
1742         device_remove_file(adev->dev, &dev_attr_current_compute_partition);
1743
1744         if (xcp_switch_supported)
1745                 device_remove_file(adev->dev,
1746                                    &dev_attr_available_compute_partition);
1747 }
1748
1749 static int amdgpu_gfx_sysfs_isolation_shader_init(struct amdgpu_device *adev)
1750 {
1751         int r;
1752
1753         r = device_create_file(adev->dev, &dev_attr_enforce_isolation);
1754         if (r)
1755                 return r;
1756         if (adev->gfx.enable_cleaner_shader)
1757                 r = device_create_file(adev->dev, &dev_attr_run_cleaner_shader);
1758
1759         return r;
1760 }
1761
1762 static void amdgpu_gfx_sysfs_isolation_shader_fini(struct amdgpu_device *adev)
1763 {
1764         device_remove_file(adev->dev, &dev_attr_enforce_isolation);
1765         if (adev->gfx.enable_cleaner_shader)
1766                 device_remove_file(adev->dev, &dev_attr_run_cleaner_shader);
1767 }
1768
1769 static int amdgpu_gfx_sysfs_reset_mask_init(struct amdgpu_device *adev)
1770 {
1771         int r = 0;
1772
1773         if (!amdgpu_gpu_recovery)
1774                 return r;
1775
1776         if (adev->gfx.num_gfx_rings) {
1777                 r = device_create_file(adev->dev, &dev_attr_gfx_reset_mask);
1778                 if (r)
1779                         return r;
1780         }
1781
1782         if (adev->gfx.num_compute_rings) {
1783                 r = device_create_file(adev->dev, &dev_attr_compute_reset_mask);
1784                 if (r)
1785                         return r;
1786         }
1787
1788         return r;
1789 }
1790
1791 static void amdgpu_gfx_sysfs_reset_mask_fini(struct amdgpu_device *adev)
1792 {
1793         if (!amdgpu_gpu_recovery)
1794                 return;
1795
1796         if (adev->gfx.num_gfx_rings)
1797                 device_remove_file(adev->dev, &dev_attr_gfx_reset_mask);
1798
1799         if (adev->gfx.num_compute_rings)
1800                 device_remove_file(adev->dev, &dev_attr_compute_reset_mask);
1801 }
1802
1803 int amdgpu_gfx_sysfs_init(struct amdgpu_device *adev)
1804 {
1805         int r;
1806
1807         r = amdgpu_gfx_sysfs_xcp_init(adev);
1808         if (r) {
1809                 dev_err(adev->dev, "failed to create xcp sysfs files");
1810                 return r;
1811         }
1812
1813         r = amdgpu_gfx_sysfs_isolation_shader_init(adev);
1814         if (r)
1815                 dev_err(adev->dev, "failed to create isolation sysfs files");
1816
1817         r = amdgpu_gfx_sysfs_reset_mask_init(adev);
1818         if (r)
1819                 dev_err(adev->dev, "failed to create reset mask sysfs files");
1820
1821         return r;
1822 }
1823
1824 void amdgpu_gfx_sysfs_fini(struct amdgpu_device *adev)
1825 {
1826         if (adev->dev->kobj.sd) {
1827                 amdgpu_gfx_sysfs_xcp_fini(adev);
1828                 amdgpu_gfx_sysfs_isolation_shader_fini(adev);
1829                 amdgpu_gfx_sysfs_reset_mask_fini(adev);
1830         }
1831 }
1832
1833 int amdgpu_gfx_cleaner_shader_sw_init(struct amdgpu_device *adev,
1834                                       unsigned int cleaner_shader_size)
1835 {
1836         if (!adev->gfx.enable_cleaner_shader)
1837                 return -EOPNOTSUPP;
1838
1839         return amdgpu_bo_create_kernel(adev, cleaner_shader_size, PAGE_SIZE,
1840                                        AMDGPU_GEM_DOMAIN_VRAM | AMDGPU_GEM_DOMAIN_GTT,
1841                                        &adev->gfx.cleaner_shader_obj,
1842                                        &adev->gfx.cleaner_shader_gpu_addr,
1843                                        (void **)&adev->gfx.cleaner_shader_cpu_ptr);
1844 }
1845
1846 void amdgpu_gfx_cleaner_shader_sw_fini(struct amdgpu_device *adev)
1847 {
1848         if (!adev->gfx.enable_cleaner_shader)
1849                 return;
1850
1851         amdgpu_bo_free_kernel(&adev->gfx.cleaner_shader_obj,
1852                               &adev->gfx.cleaner_shader_gpu_addr,
1853                               (void **)&adev->gfx.cleaner_shader_cpu_ptr);
1854 }
1855
1856 void amdgpu_gfx_cleaner_shader_init(struct amdgpu_device *adev,
1857                                     unsigned int cleaner_shader_size,
1858                                     const void *cleaner_shader_ptr)
1859 {
1860         if (!adev->gfx.enable_cleaner_shader)
1861                 return;
1862
1863         if (adev->gfx.cleaner_shader_cpu_ptr && cleaner_shader_ptr)
1864                 memcpy_toio(adev->gfx.cleaner_shader_cpu_ptr, cleaner_shader_ptr,
1865                             cleaner_shader_size);
1866 }
1867
1868 /**
1869  * amdgpu_gfx_kfd_sch_ctrl - Control the KFD scheduler from the KGD (Graphics Driver)
1870  * @adev: amdgpu_device pointer
1871  * @idx: Index of the scheduler to control
1872  * @enable: Whether to enable or disable the KFD scheduler
1873  *
1874  * This function is used to control the KFD (Kernel Fusion Driver) scheduler
1875  * from the KGD. It is part of the cleaner shader feature. This function plays
1876  * a key role in enforcing process isolation on the GPU.
1877  *
1878  * The function uses a reference count mechanism (kfd_sch_req_count) to keep
1879  * track of the number of requests to enable the KFD scheduler. When a request
1880  * to enable the KFD scheduler is made, the reference count is decremented.
1881  * When the reference count reaches zero, a delayed work is scheduled to
1882  * enforce isolation after a delay of GFX_SLICE_PERIOD.
1883  *
1884  * When a request to disable the KFD scheduler is made, the function first
1885  * checks if the reference count is zero. If it is, it cancels the delayed work
1886  * for enforcing isolation and checks if the KFD scheduler is active. If the
1887  * KFD scheduler is active, it sends a request to stop the KFD scheduler and
1888  * sets the KFD scheduler state to inactive. Then, it increments the reference
1889  * count.
1890  *
1891  * The function is synchronized using the kfd_sch_mutex to ensure that the KFD
1892  * scheduler state and reference count are updated atomically.
1893  *
1894  * Note: If the reference count is already zero when a request to enable the
1895  * KFD scheduler is made, it means there's an imbalance bug somewhere. The
1896  * function triggers a warning in this case.
1897  */
1898 static void amdgpu_gfx_kfd_sch_ctrl(struct amdgpu_device *adev, u32 idx,
1899                                     bool enable)
1900 {
1901         mutex_lock(&adev->gfx.kfd_sch_mutex);
1902
1903         if (enable) {
1904                 /* If the count is already 0, it means there's an imbalance bug somewhere.
1905                  * Note that the bug may be in a different caller than the one which triggers the
1906                  * WARN_ON_ONCE.
1907                  */
1908                 if (WARN_ON_ONCE(adev->gfx.kfd_sch_req_count[idx] == 0)) {
1909                         dev_err(adev->dev, "Attempted to enable KFD scheduler when reference count is already zero\n");
1910                         goto unlock;
1911                 }
1912
1913                 adev->gfx.kfd_sch_req_count[idx]--;
1914
1915                 if (adev->gfx.kfd_sch_req_count[idx] == 0 &&
1916                     adev->gfx.kfd_sch_inactive[idx]) {
1917                         schedule_delayed_work(&adev->gfx.enforce_isolation[idx].work,
1918                                               msecs_to_jiffies(adev->gfx.enforce_isolation_time[idx]));
1919                 }
1920         } else {
1921                 if (adev->gfx.kfd_sch_req_count[idx] == 0) {
1922                         cancel_delayed_work_sync(&adev->gfx.enforce_isolation[idx].work);
1923                         if (!adev->gfx.kfd_sch_inactive[idx]) {
1924                                 amdgpu_amdkfd_stop_sched(adev, idx);
1925                                 adev->gfx.kfd_sch_inactive[idx] = true;
1926                         }
1927                 }
1928
1929                 adev->gfx.kfd_sch_req_count[idx]++;
1930         }
1931
1932 unlock:
1933         mutex_unlock(&adev->gfx.kfd_sch_mutex);
1934 }
1935
1936 /**
1937  * amdgpu_gfx_enforce_isolation_handler - work handler for enforcing shader isolation
1938  *
1939  * @work: work_struct.
1940  *
1941  * This function is the work handler for enforcing shader isolation on AMD GPUs.
1942  * It counts the number of emitted fences for each GFX and compute ring. If there
1943  * are any fences, it schedules the `enforce_isolation_work` to be run after a
1944  * delay of `GFX_SLICE_PERIOD`. If there are no fences, it signals the Kernel Fusion
1945  * Driver (KFD) to resume the runqueue. The function is synchronized using the
1946  * `enforce_isolation_mutex`.
1947  */
1948 void amdgpu_gfx_enforce_isolation_handler(struct work_struct *work)
1949 {
1950         struct amdgpu_isolation_work *isolation_work =
1951                 container_of(work, struct amdgpu_isolation_work, work.work);
1952         struct amdgpu_device *adev = isolation_work->adev;
1953         u32 i, idx, fences = 0;
1954
1955         if (isolation_work->xcp_id == AMDGPU_XCP_NO_PARTITION)
1956                 idx = 0;
1957         else
1958                 idx = isolation_work->xcp_id;
1959
1960         if (idx >= MAX_XCP)
1961                 return;
1962
1963         mutex_lock(&adev->enforce_isolation_mutex);
1964         for (i = 0; i < AMDGPU_MAX_GFX_RINGS; ++i) {
1965                 if (isolation_work->xcp_id == adev->gfx.gfx_ring[i].xcp_id)
1966                         fences += amdgpu_fence_count_emitted(&adev->gfx.gfx_ring[i]);
1967         }
1968         for (i = 0; i < (AMDGPU_MAX_COMPUTE_RINGS * AMDGPU_MAX_GC_INSTANCES); ++i) {
1969                 if (isolation_work->xcp_id == adev->gfx.compute_ring[i].xcp_id)
1970                         fences += amdgpu_fence_count_emitted(&adev->gfx.compute_ring[i]);
1971         }
1972         if (fences) {
1973                 /* we've already had our timeslice, so let's wrap this up */
1974                 schedule_delayed_work(&adev->gfx.enforce_isolation[idx].work,
1975                                       msecs_to_jiffies(1));
1976         } else {
1977                 /* Tell KFD to resume the runqueue */
1978                 if (adev->kfd.init_complete) {
1979                         WARN_ON_ONCE(!adev->gfx.kfd_sch_inactive[idx]);
1980                         WARN_ON_ONCE(adev->gfx.kfd_sch_req_count[idx]);
1981                                 amdgpu_amdkfd_start_sched(adev, idx);
1982                                 adev->gfx.kfd_sch_inactive[idx] = false;
1983                 }
1984         }
1985         mutex_unlock(&adev->enforce_isolation_mutex);
1986 }
1987
1988 /**
1989  * amdgpu_gfx_enforce_isolation_wait_for_kfd - Manage KFD wait period for process isolation
1990  * @adev: amdgpu_device pointer
1991  * @idx: Index of the GPU partition
1992  *
1993  * When kernel submissions come in, the jobs are given a time slice and once
1994  * that time slice is up, if there are KFD user queues active, kernel
1995  * submissions are blocked until KFD has had its time slice. Once the KFD time
1996  * slice is up, KFD user queues are preempted and kernel submissions are
1997  * unblocked and allowed to run again.
1998  */
1999 static void
2000 amdgpu_gfx_enforce_isolation_wait_for_kfd(struct amdgpu_device *adev,
2001                                           u32 idx)
2002 {
2003         unsigned long cjiffies;
2004         bool wait = false;
2005
2006         mutex_lock(&adev->enforce_isolation_mutex);
2007         if (adev->enforce_isolation[idx]) {
2008                 /* set the initial values if nothing is set */
2009                 if (!adev->gfx.enforce_isolation_jiffies[idx]) {
2010                         adev->gfx.enforce_isolation_jiffies[idx] = jiffies;
2011                         adev->gfx.enforce_isolation_time[idx] = GFX_SLICE_PERIOD_MS;
2012                 }
2013                 /* Make sure KFD gets a chance to run */
2014                 if (amdgpu_amdkfd_compute_active(adev, idx)) {
2015                         cjiffies = jiffies;
2016                         if (time_after(cjiffies, adev->gfx.enforce_isolation_jiffies[idx])) {
2017                                 cjiffies -= adev->gfx.enforce_isolation_jiffies[idx];
2018                                 if ((jiffies_to_msecs(cjiffies) >= GFX_SLICE_PERIOD_MS)) {
2019                                         /* if our time is up, let KGD work drain before scheduling more */
2020                                         wait = true;
2021                                         /* reset the timer period */
2022                                         adev->gfx.enforce_isolation_time[idx] = GFX_SLICE_PERIOD_MS;
2023                                 } else {
2024                                         /* set the timer period to what's left in our time slice */
2025                                         adev->gfx.enforce_isolation_time[idx] =
2026                                                 GFX_SLICE_PERIOD_MS - jiffies_to_msecs(cjiffies);
2027                                 }
2028                         } else {
2029                                 /* if jiffies wrap around we will just wait a little longer */
2030                                 adev->gfx.enforce_isolation_jiffies[idx] = jiffies;
2031                         }
2032                 } else {
2033                         /* if there is no KFD work, then set the full slice period */
2034                         adev->gfx.enforce_isolation_jiffies[idx] = jiffies;
2035                         adev->gfx.enforce_isolation_time[idx] = GFX_SLICE_PERIOD_MS;
2036                 }
2037         }
2038         mutex_unlock(&adev->enforce_isolation_mutex);
2039
2040         if (wait)
2041                 msleep(GFX_SLICE_PERIOD_MS);
2042 }
2043
2044 /**
2045  * amdgpu_gfx_enforce_isolation_ring_begin_use - Begin use of a ring with enforced isolation
2046  * @ring: Pointer to the amdgpu_ring structure
2047  *
2048  * Ring begin_use helper implementation for gfx which serializes access to the
2049  * gfx IP between kernel submission IOCTLs and KFD user queues when isolation
2050  * enforcement is enabled. The kernel submission IOCTLs and KFD user queues
2051  * each get a time slice when both are active.
2052  */
2053 void amdgpu_gfx_enforce_isolation_ring_begin_use(struct amdgpu_ring *ring)
2054 {
2055         struct amdgpu_device *adev = ring->adev;
2056         u32 idx;
2057         bool sched_work = false;
2058
2059         if (!adev->gfx.enable_cleaner_shader)
2060                 return;
2061
2062         if (ring->xcp_id == AMDGPU_XCP_NO_PARTITION)
2063                 idx = 0;
2064         else
2065                 idx = ring->xcp_id;
2066
2067         if (idx >= MAX_XCP)
2068                 return;
2069
2070         /* Don't submit more work until KFD has had some time */
2071         amdgpu_gfx_enforce_isolation_wait_for_kfd(adev, idx);
2072
2073         mutex_lock(&adev->enforce_isolation_mutex);
2074         if (adev->enforce_isolation[idx]) {
2075                 if (adev->kfd.init_complete)
2076                         sched_work = true;
2077         }
2078         mutex_unlock(&adev->enforce_isolation_mutex);
2079
2080         if (sched_work)
2081                 amdgpu_gfx_kfd_sch_ctrl(adev, idx, false);
2082 }
2083
2084 /**
2085  * amdgpu_gfx_enforce_isolation_ring_end_use - End use of a ring with enforced isolation
2086  * @ring: Pointer to the amdgpu_ring structure
2087  *
2088  * Ring end_use helper implementation for gfx which serializes access to the
2089  * gfx IP between kernel submission IOCTLs and KFD user queues when isolation
2090  * enforcement is enabled. The kernel submission IOCTLs and KFD user queues
2091  * each get a time slice when both are active.
2092  */
2093 void amdgpu_gfx_enforce_isolation_ring_end_use(struct amdgpu_ring *ring)
2094 {
2095         struct amdgpu_device *adev = ring->adev;
2096         u32 idx;
2097         bool sched_work = false;
2098
2099         if (!adev->gfx.enable_cleaner_shader)
2100                 return;
2101
2102         if (ring->xcp_id == AMDGPU_XCP_NO_PARTITION)
2103                 idx = 0;
2104         else
2105                 idx = ring->xcp_id;
2106
2107         if (idx >= MAX_XCP)
2108                 return;
2109
2110         mutex_lock(&adev->enforce_isolation_mutex);
2111         if (adev->enforce_isolation[idx]) {
2112                 if (adev->kfd.init_complete)
2113                         sched_work = true;
2114         }
2115         mutex_unlock(&adev->enforce_isolation_mutex);
2116
2117         if (sched_work)
2118                 amdgpu_gfx_kfd_sch_ctrl(adev, idx, true);
2119 }
2120
2121 /*
2122  * debugfs for to enable/disable gfx job submission to specific core.
2123  */
2124 #if defined(CONFIG_DEBUG_FS)
2125 static int amdgpu_debugfs_gfx_sched_mask_set(void *data, u64 val)
2126 {
2127         struct amdgpu_device *adev = (struct amdgpu_device *)data;
2128         u32 i;
2129         u64 mask = 0;
2130         struct amdgpu_ring *ring;
2131
2132         if (!adev)
2133                 return -ENODEV;
2134
2135         mask = (1ULL << adev->gfx.num_gfx_rings) - 1;
2136         if ((val & mask) == 0)
2137                 return -EINVAL;
2138
2139         for (i = 0; i < adev->gfx.num_gfx_rings; ++i) {
2140                 ring = &adev->gfx.gfx_ring[i];
2141                 if (val & (1 << i))
2142                         ring->sched.ready = true;
2143                 else
2144                         ring->sched.ready = false;
2145         }
2146         /* publish sched.ready flag update effective immediately across smp */
2147         smp_rmb();
2148         return 0;
2149 }
2150
2151 static int amdgpu_debugfs_gfx_sched_mask_get(void *data, u64 *val)
2152 {
2153         struct amdgpu_device *adev = (struct amdgpu_device *)data;
2154         u32 i;
2155         u64 mask = 0;
2156         struct amdgpu_ring *ring;
2157
2158         if (!adev)
2159                 return -ENODEV;
2160         for (i = 0; i < adev->gfx.num_gfx_rings; ++i) {
2161                 ring = &adev->gfx.gfx_ring[i];
2162                 if (ring->sched.ready)
2163                         mask |= 1ULL << i;
2164         }
2165
2166         *val = mask;
2167         return 0;
2168 }
2169
2170 DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_debugfs_gfx_sched_mask_fops,
2171                          amdgpu_debugfs_gfx_sched_mask_get,
2172                          amdgpu_debugfs_gfx_sched_mask_set, "%llx\n");
2173
2174 #endif
2175
2176 void amdgpu_debugfs_gfx_sched_mask_init(struct amdgpu_device *adev)
2177 {
2178 #if defined(CONFIG_DEBUG_FS)
2179         struct drm_minor *minor = adev_to_drm(adev)->primary;
2180         struct dentry *root = minor->debugfs_root;
2181         char name[32];
2182
2183         if (!(adev->gfx.num_gfx_rings > 1))
2184                 return;
2185         sprintf(name, "amdgpu_gfx_sched_mask");
2186         debugfs_create_file(name, 0600, root, adev,
2187                             &amdgpu_debugfs_gfx_sched_mask_fops);
2188 #endif
2189 }
2190
2191 /*
2192  * debugfs for to enable/disable compute job submission to specific core.
2193  */
2194 #if defined(CONFIG_DEBUG_FS)
2195 static int amdgpu_debugfs_compute_sched_mask_set(void *data, u64 val)
2196 {
2197         struct amdgpu_device *adev = (struct amdgpu_device *)data;
2198         u32 i;
2199         u64 mask = 0;
2200         struct amdgpu_ring *ring;
2201
2202         if (!adev)
2203                 return -ENODEV;
2204
2205         mask = (1ULL << adev->gfx.num_compute_rings) - 1;
2206         if ((val & mask) == 0)
2207                 return -EINVAL;
2208
2209         for (i = 0; i < adev->gfx.num_compute_rings; ++i) {
2210                 ring = &adev->gfx.compute_ring[i];
2211                 if (val & (1 << i))
2212                         ring->sched.ready = true;
2213                 else
2214                         ring->sched.ready = false;
2215         }
2216
2217         /* publish sched.ready flag update effective immediately across smp */
2218         smp_rmb();
2219         return 0;
2220 }
2221
2222 static int amdgpu_debugfs_compute_sched_mask_get(void *data, u64 *val)
2223 {
2224         struct amdgpu_device *adev = (struct amdgpu_device *)data;
2225         u32 i;
2226         u64 mask = 0;
2227         struct amdgpu_ring *ring;
2228
2229         if (!adev)
2230                 return -ENODEV;
2231         for (i = 0; i < adev->gfx.num_compute_rings; ++i) {
2232                 ring = &adev->gfx.compute_ring[i];
2233                 if (ring->sched.ready)
2234                         mask |= 1ULL << i;
2235         }
2236
2237         *val = mask;
2238         return 0;
2239 }
2240
2241 DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_debugfs_compute_sched_mask_fops,
2242                          amdgpu_debugfs_compute_sched_mask_get,
2243                          amdgpu_debugfs_compute_sched_mask_set, "%llx\n");
2244
2245 #endif
2246
2247 void amdgpu_debugfs_compute_sched_mask_init(struct amdgpu_device *adev)
2248 {
2249 #if defined(CONFIG_DEBUG_FS)
2250         struct drm_minor *minor = adev_to_drm(adev)->primary;
2251         struct dentry *root = minor->debugfs_root;
2252         char name[32];
2253
2254         if (!(adev->gfx.num_compute_rings > 1))
2255                 return;
2256         sprintf(name, "amdgpu_compute_sched_mask");
2257         debugfs_create_file(name, 0600, root, adev,
2258                             &amdgpu_debugfs_compute_sched_mask_fops);
2259 #endif
2260 }