drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c

   1 /*
   2  * Copyright 2014 Advanced Micro Devices, Inc.
   3  * Copyright 2008 Red Hat Inc.
   4  * Copyright 2009 Jerome Glisse.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22  * OTHER DEALINGS IN THE SOFTWARE.
  23  *
  24  */
  25
  26 #include "amdgpu.h"
  27 #include "amdgpu_gfx.h"
  28 #include "amdgpu_rlc.h"
  29 #include "amdgpu_ras.h"
  30
  31 /* delay 0.1 second to enable gfx off feature */
  32 #define GFX_OFF_DELAY_ENABLE         msecs_to_jiffies(100)
  33
  34 #define GFX_OFF_NO_DELAY 0
  35
  36 /*
  37  * GPU GFX IP block helpers function.
  38  */
  39
  40 int amdgpu_gfx_mec_queue_to_bit(struct amdgpu_device *adev, int mec,
  41                                 int pipe, int queue)
  42 {
  43         int bit = 0;
  44
  45         bit += mec * adev->gfx.mec.num_pipe_per_mec
  46                 * adev->gfx.mec.num_queue_per_pipe;
  47         bit += pipe * adev->gfx.mec.num_queue_per_pipe;
  48         bit += queue;
  49
  50         return bit;
  51 }
  52
  53 void amdgpu_queue_mask_bit_to_mec_queue(struct amdgpu_device *adev, int bit,
  54                                  int *mec, int *pipe, int *queue)
  55 {
  56         *queue = bit % adev->gfx.mec.num_queue_per_pipe;
  57         *pipe = (bit / adev->gfx.mec.num_queue_per_pipe)
  58                 % adev->gfx.mec.num_pipe_per_mec;
  59         *mec = (bit / adev->gfx.mec.num_queue_per_pipe)
  60                / adev->gfx.mec.num_pipe_per_mec;
  61
  62 }
  63
  64 bool amdgpu_gfx_is_mec_queue_enabled(struct amdgpu_device *adev,
  65                                      int mec, int pipe, int queue)
  66 {
  67         return test_bit(amdgpu_gfx_mec_queue_to_bit(adev, mec, pipe, queue),
  68                         adev->gfx.mec.queue_bitmap);
  69 }
  70
  71 int amdgpu_gfx_me_queue_to_bit(struct amdgpu_device *adev,
  72                                int me, int pipe, int queue)
  73 {
  74         int bit = 0;
  75
  76         bit += me * adev->gfx.me.num_pipe_per_me
  77                 * adev->gfx.me.num_queue_per_pipe;
  78         bit += pipe * adev->gfx.me.num_queue_per_pipe;
  79         bit += queue;
  80
  81         return bit;
  82 }
  83
  84 void amdgpu_gfx_bit_to_me_queue(struct amdgpu_device *adev, int bit,
  85                                 int *me, int *pipe, int *queue)
  86 {
  87         *queue = bit % adev->gfx.me.num_queue_per_pipe;
  88         *pipe = (bit / adev->gfx.me.num_queue_per_pipe)
  89                 % adev->gfx.me.num_pipe_per_me;
  90         *me = (bit / adev->gfx.me.num_queue_per_pipe)
  91                 / adev->gfx.me.num_pipe_per_me;
  92 }
  93
  94 bool amdgpu_gfx_is_me_queue_enabled(struct amdgpu_device *adev,
  95                                     int me, int pipe, int queue)
  96 {
  97         return test_bit(amdgpu_gfx_me_queue_to_bit(adev, me, pipe, queue),
  98                         adev->gfx.me.queue_bitmap);
  99 }
 100
 101 /**
 102  * amdgpu_gfx_scratch_get - Allocate a scratch register
 103  *
 104  * @adev: amdgpu_device pointer
 105  * @reg: scratch register mmio offset
 106  *
 107  * Allocate a CP scratch register for use by the driver (all asics).
 108  * Returns 0 on success or -EINVAL on failure.
 109  */
 110 int amdgpu_gfx_scratch_get(struct amdgpu_device *adev, uint32_t *reg)
 111 {
 112         int i;
 113
 114         i = ffs(adev->gfx.scratch.free_mask);
 115         if (i != 0 && i <= adev->gfx.scratch.num_reg) {
 116                 i--;
 117                 adev->gfx.scratch.free_mask &= ~(1u << i);
 118                 *reg = adev->gfx.scratch.reg_base + i;
 119                 return 0;
 120         }
 121         return -EINVAL;
 122 }
 123
 124 /**
 125  * amdgpu_gfx_scratch_free - Free a scratch register
 126  *
 127  * @adev: amdgpu_device pointer
 128  * @reg: scratch register mmio offset
 129  *
 130  * Free a CP scratch register allocated for use by the driver (all asics)
 131  */
 132 void amdgpu_gfx_scratch_free(struct amdgpu_device *adev, uint32_t reg)
 133 {
 134         adev->gfx.scratch.free_mask |= 1u << (reg - adev->gfx.scratch.reg_base);
 135 }
 136
 137 /**
 138  * amdgpu_gfx_parse_disable_cu - Parse the disable_cu module parameter
 139  *
 140  * @mask: array in which the per-shader array disable masks will be stored
 141  * @max_se: number of SEs
 142  * @max_sh: number of SHs
 143  *
 144  * The bitmask of CUs to be disabled in the shader array determined by se and
 145  * sh is stored in mask[se * max_sh + sh].
 146  */
 147 void amdgpu_gfx_parse_disable_cu(unsigned *mask, unsigned max_se, unsigned max_sh)
 148 {
 149         unsigned se, sh, cu;
 150         const char *p;
 151
 152         memset(mask, 0, sizeof(*mask) * max_se * max_sh);
 153
 154         if (!amdgpu_disable_cu || !*amdgpu_disable_cu)
 155                 return;
 156
 157         p = amdgpu_disable_cu;
 158         for (;;) {
 159                 char *next;
 160                 int ret = sscanf(p, "%u.%u.%u", &se, &sh, &cu);
 161                 if (ret < 3) {
 162                         DRM_ERROR("amdgpu: could not parse disable_cu\n");
 163                         return;
 164                 }
 165
 166                 if (se < max_se && sh < max_sh && cu < 16) {
 167                         DRM_INFO("amdgpu: disabling CU %u.%u.%u\n", se, sh, cu);
 168                         mask[se * max_sh + sh] |= 1u << cu;
 169                 } else {
 170                         DRM_ERROR("amdgpu: disable_cu %u.%u.%u is out of range\n",
 171                                   se, sh, cu);
 172                 }
 173
 174                 next = strchr(p, ',');
 175                 if (!next)
 176                         break;
 177                 p = next + 1;
 178         }
 179 }
 180
 181 static bool amdgpu_gfx_is_multipipe_capable(struct amdgpu_device *adev)
 182 {
 183         if (amdgpu_compute_multipipe != -1) {
 184                 DRM_INFO("amdgpu: forcing compute pipe policy %d\n",
 185                          amdgpu_compute_multipipe);
 186                 return amdgpu_compute_multipipe == 1;
 187         }
 188
 189         /* FIXME: spreading the queues across pipes causes perf regressions
 190          * on POLARIS11 compute workloads */
 191         if (adev->asic_type == CHIP_POLARIS11)
 192                 return false;
 193
 194         return adev->gfx.mec.num_mec > 1;
 195 }
 196
 197 bool amdgpu_gfx_is_high_priority_compute_queue(struct amdgpu_device *adev,
 198                                                struct amdgpu_ring *ring)
 199 {
 200         /* Policy: use 1st queue as high priority compute queue if we
 201          * have more than one compute queue.
 202          */
 203         if (adev->gfx.num_compute_rings > 1 &&
 204             ring == &adev->gfx.compute_ring[0])
 205                 return true;
 206
 207         return false;
 208 }
 209
 210 void amdgpu_gfx_compute_queue_acquire(struct amdgpu_device *adev)
 211 {
 212         int i, queue, pipe;
 213         bool multipipe_policy = amdgpu_gfx_is_multipipe_capable(adev);
 214         int max_queues_per_mec = min(adev->gfx.mec.num_pipe_per_mec *
 215                                      adev->gfx.mec.num_queue_per_pipe,
 216                                      adev->gfx.num_compute_rings);
 217
 218         if (multipipe_policy) {
 219                 /* policy: make queues evenly cross all pipes on MEC1 only */
 220                 for (i = 0; i < max_queues_per_mec; i++) {
 221                         pipe = i % adev->gfx.mec.num_pipe_per_mec;
 222                         queue = (i / adev->gfx.mec.num_pipe_per_mec) %
 223                                 adev->gfx.mec.num_queue_per_pipe;
 224
 225                         set_bit(pipe * adev->gfx.mec.num_queue_per_pipe + queue,
 226                                         adev->gfx.mec.queue_bitmap);
 227                 }
 228         } else {
 229                 /* policy: amdgpu owns all queues in the given pipe */
 230                 for (i = 0; i < max_queues_per_mec; ++i)
 231                         set_bit(i, adev->gfx.mec.queue_bitmap);
 232         }
 233
 234         dev_dbg(adev->dev, "mec queue bitmap weight=%d\n", bitmap_weight(adev->gfx.mec.queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES));
 235 }
 236
 237 void amdgpu_gfx_graphics_queue_acquire(struct amdgpu_device *adev)
 238 {
 239         int i, queue, me;
 240
 241         for (i = 0; i < AMDGPU_MAX_GFX_QUEUES; ++i) {
 242                 queue = i % adev->gfx.me.num_queue_per_pipe;
 243                 me = (i / adev->gfx.me.num_queue_per_pipe)
 244                       / adev->gfx.me.num_pipe_per_me;
 245
 246                 if (me >= adev->gfx.me.num_me)
 247                         break;
 248                 /* policy: amdgpu owns the first queue per pipe at this stage
 249                  * will extend to mulitple queues per pipe later */
 250                 if (me == 0 && queue < 1)
 251                         set_bit(i, adev->gfx.me.queue_bitmap);
 252         }
 253
 254         /* update the number of active graphics rings */
 255         adev->gfx.num_gfx_rings =
 256                 bitmap_weight(adev->gfx.me.queue_bitmap, AMDGPU_MAX_GFX_QUEUES);
 257 }
 258
 259 static int amdgpu_gfx_kiq_acquire(struct amdgpu_device *adev,
 260                                   struct amdgpu_ring *ring)
 261 {
 262         int queue_bit;
 263         int mec, pipe, queue;
 264
 265         queue_bit = adev->gfx.mec.num_mec
 266                     * adev->gfx.mec.num_pipe_per_mec
 267                     * adev->gfx.mec.num_queue_per_pipe;
 268
 269         while (queue_bit-- >= 0) {
 270                 if (test_bit(queue_bit, adev->gfx.mec.queue_bitmap))
 271                         continue;
 272
 273                 amdgpu_queue_mask_bit_to_mec_queue(adev, queue_bit, &mec, &pipe, &queue);
 274
 275                 /*
 276                  * 1. Using pipes 2/3 from MEC 2 seems cause problems.
 277                  * 2. It must use queue id 0, because CGPG_IDLE/SAVE/LOAD/RUN
 278                  * only can be issued on queue 0.
 279                  */
 280                 if ((mec == 1 && pipe > 1) || queue != 0)
 281                         continue;
 282
 283                 ring->me = mec + 1;
 284                 ring->pipe = pipe;
 285                 ring->queue = queue;
 286
 287                 return 0;
 288         }
 289
 290         dev_err(adev->dev, "Failed to find a queue for KIQ\n");
 291         return -EINVAL;
 292 }
 293
 294 int amdgpu_gfx_kiq_init_ring(struct amdgpu_device *adev,
 295                              struct amdgpu_ring *ring,
 296                              struct amdgpu_irq_src *irq)
 297 {
 298         struct amdgpu_kiq *kiq = &adev->gfx.kiq;
 299         int r = 0;
 300
 301         spin_lock_init(&kiq->ring_lock);
 302
 303         ring->adev = NULL;
 304         ring->ring_obj = NULL;
 305         ring->use_doorbell = true;
 306         ring->doorbell_index = adev->doorbell_index.kiq;
 307
 308         r = amdgpu_gfx_kiq_acquire(adev, ring);
 309         if (r)
 310                 return r;
 311
 312         ring->eop_gpu_addr = kiq->eop_gpu_addr;
 313         ring->no_scheduler = true;
 314         sprintf(ring->name, "kiq_%d.%d.%d", ring->me, ring->pipe, ring->queue);
 315         r = amdgpu_ring_init(adev, ring, 1024, irq, AMDGPU_CP_KIQ_IRQ_DRIVER0,
 316                              AMDGPU_RING_PRIO_DEFAULT, NULL);
 317         if (r)
 318                 dev_warn(adev->dev, "(%d) failed to init kiq ring\n", r);
 319
 320         return r;
 321 }
 322
 323 void amdgpu_gfx_kiq_free_ring(struct amdgpu_ring *ring)
 324 {
 325         amdgpu_ring_fini(ring);
 326 }
 327
 328 void amdgpu_gfx_kiq_fini(struct amdgpu_device *adev)
 329 {
 330         struct amdgpu_kiq *kiq = &adev->gfx.kiq;
 331
 332         amdgpu_bo_free_kernel(&kiq->eop_obj, &kiq->eop_gpu_addr, NULL);
 333 }
 334
 335 int amdgpu_gfx_kiq_init(struct amdgpu_device *adev,
 336                         unsigned hpd_size)
 337 {
 338         int r;
 339         u32 *hpd;
 340         struct amdgpu_kiq *kiq = &adev->gfx.kiq;
 341
 342         r = amdgpu_bo_create_kernel(adev, hpd_size, PAGE_SIZE,
 343                                     AMDGPU_GEM_DOMAIN_GTT, &kiq->eop_obj,
 344                                     &kiq->eop_gpu_addr, (void **)&hpd);
 345         if (r) {
 346                 dev_warn(adev->dev, "failed to create KIQ bo (%d).\n", r);
 347                 return r;
 348         }
 349
 350         memset(hpd, 0, hpd_size);
 351
 352         r = amdgpu_bo_reserve(kiq->eop_obj, true);
 353         if (unlikely(r != 0))
 354                 dev_warn(adev->dev, "(%d) reserve kiq eop bo failed\n", r);
 355         amdgpu_bo_kunmap(kiq->eop_obj);
 356         amdgpu_bo_unreserve(kiq->eop_obj);
 357
 358         return 0;
 359 }
 360
 361 /* create MQD for each compute/gfx queue */
 362 int amdgpu_gfx_mqd_sw_init(struct amdgpu_device *adev,
 363                            unsigned mqd_size)
 364 {
 365         struct amdgpu_ring *ring = NULL;
 366         int r, i;
 367
 368         /* create MQD for KIQ */
 369         ring = &adev->gfx.kiq.ring;
 370         if (!ring->mqd_obj) {
 371                 /* originaly the KIQ MQD is put in GTT domain, but for SRIOV VRAM domain is a must
 372                  * otherwise hypervisor trigger SAVE_VF fail after driver unloaded which mean MQD
 373                  * deallocated and gart_unbind, to strict diverage we decide to use VRAM domain for
 374                  * KIQ MQD no matter SRIOV or Bare-metal
 375                  */
 376                 r = amdgpu_bo_create_kernel(adev, mqd_size, PAGE_SIZE,
 377                                             AMDGPU_GEM_DOMAIN_VRAM, &ring->mqd_obj,
 378                                             &ring->mqd_gpu_addr, &ring->mqd_ptr);
 379                 if (r) {
 380                         dev_warn(adev->dev, "failed to create ring mqd ob (%d)", r);
 381                         return r;
 382                 }
 383
 384                 /* prepare MQD backup */
 385                 adev->gfx.mec.mqd_backup[AMDGPU_MAX_COMPUTE_RINGS] = kmalloc(mqd_size, GFP_KERNEL);
 386                 if (!adev->gfx.mec.mqd_backup[AMDGPU_MAX_COMPUTE_RINGS])
 387                                 dev_warn(adev->dev, "no memory to create MQD backup for ring %s\n", ring->name);
 388         }
 389
 390         if (adev->asic_type >= CHIP_NAVI10 && amdgpu_async_gfx_ring) {
 391                 /* create MQD for each KGQ */
 392                 for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
 393                         ring = &adev->gfx.gfx_ring[i];
 394                         if (!ring->mqd_obj) {
 395                                 r = amdgpu_bo_create_kernel(adev, mqd_size, PAGE_SIZE,
 396                                                             AMDGPU_GEM_DOMAIN_GTT, &ring->mqd_obj,
 397                                                             &ring->mqd_gpu_addr, &ring->mqd_ptr);
 398                                 if (r) {
 399                                         dev_warn(adev->dev, "failed to create ring mqd bo (%d)", r);
 400                                         return r;
 401                                 }
 402
 403                                 /* prepare MQD backup */
 404                                 adev->gfx.me.mqd_backup[i] = kmalloc(mqd_size, GFP_KERNEL);
 405                                 if (!adev->gfx.me.mqd_backup[i])
 406                                         dev_warn(adev->dev, "no memory to create MQD backup for ring %s\n", ring->name);
 407                         }
 408                 }
 409         }
 410
 411         /* create MQD for each KCQ */
 412         for (i = 0; i < adev->gfx.num_compute_rings; i++) {
 413                 ring = &adev->gfx.compute_ring[i];
 414                 if (!ring->mqd_obj) {
 415                         r = amdgpu_bo_create_kernel(adev, mqd_size, PAGE_SIZE,
 416                                                     AMDGPU_GEM_DOMAIN_GTT, &ring->mqd_obj,
 417                                                     &ring->mqd_gpu_addr, &ring->mqd_ptr);
 418                         if (r) {
 419                                 dev_warn(adev->dev, "failed to create ring mqd bo (%d)", r);
 420                                 return r;
 421                         }
 422
 423                         /* prepare MQD backup */
 424                         adev->gfx.mec.mqd_backup[i] = kmalloc(mqd_size, GFP_KERNEL);
 425                         if (!adev->gfx.mec.mqd_backup[i])
 426                                 dev_warn(adev->dev, "no memory to create MQD backup for ring %s\n", ring->name);
 427                 }
 428         }
 429
 430         return 0;
 431 }
 432
 433 void amdgpu_gfx_mqd_sw_fini(struct amdgpu_device *adev)
 434 {
 435         struct amdgpu_ring *ring = NULL;
 436         int i;
 437
 438         if (adev->asic_type >= CHIP_NAVI10 && amdgpu_async_gfx_ring) {
 439                 for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
 440                         ring = &adev->gfx.gfx_ring[i];
 441                         kfree(adev->gfx.me.mqd_backup[i]);
 442                         amdgpu_bo_free_kernel(&ring->mqd_obj,
 443                                               &ring->mqd_gpu_addr,
 444                                               &ring->mqd_ptr);
 445                 }
 446         }
 447
 448         for (i = 0; i < adev->gfx.num_compute_rings; i++) {
 449                 ring = &adev->gfx.compute_ring[i];
 450                 kfree(adev->gfx.mec.mqd_backup[i]);
 451                 amdgpu_bo_free_kernel(&ring->mqd_obj,
 452                                       &ring->mqd_gpu_addr,
 453                                       &ring->mqd_ptr);
 454         }
 455
 456         ring = &adev->gfx.kiq.ring;
 457         kfree(adev->gfx.mec.mqd_backup[AMDGPU_MAX_COMPUTE_RINGS]);
 458         amdgpu_bo_free_kernel(&ring->mqd_obj,
 459                               &ring->mqd_gpu_addr,
 460                               &ring->mqd_ptr);
 461 }
 462
 463 int amdgpu_gfx_disable_kcq(struct amdgpu_device *adev)
 464 {
 465         struct amdgpu_kiq *kiq = &adev->gfx.kiq;
 466         struct amdgpu_ring *kiq_ring = &kiq->ring;
 467         int i, r;
 468
 469         if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues)
 470                 return -EINVAL;
 471
 472         spin_lock(&adev->gfx.kiq.ring_lock);
 473         if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size *
 474                                         adev->gfx.num_compute_rings)) {
 475                 spin_unlock(&adev->gfx.kiq.ring_lock);
 476                 return -ENOMEM;
 477         }
 478
 479         for (i = 0; i < adev->gfx.num_compute_rings; i++)
 480                 kiq->pmf->kiq_unmap_queues(kiq_ring, &adev->gfx.compute_ring[i],
 481                                            RESET_QUEUES, 0, 0);
 482         r = amdgpu_ring_test_helper(kiq_ring);
 483         spin_unlock(&adev->gfx.kiq.ring_lock);
 484
 485         return r;
 486 }
 487
 488 int amdgpu_queue_mask_bit_to_set_resource_bit(struct amdgpu_device *adev,
 489                                         int queue_bit)
 490 {
 491         int mec, pipe, queue;
 492         int set_resource_bit = 0;
 493
 494         amdgpu_queue_mask_bit_to_mec_queue(adev, queue_bit, &mec, &pipe, &queue);
 495
 496         set_resource_bit = mec * 4 * 8 + pipe * 8 + queue;
 497
 498         return set_resource_bit;
 499 }
 500
 501 int amdgpu_gfx_enable_kcq(struct amdgpu_device *adev)
 502 {
 503         struct amdgpu_kiq *kiq = &adev->gfx.kiq;
 504         struct amdgpu_ring *kiq_ring = &adev->gfx.kiq.ring;
 505         uint64_t queue_mask = 0;
 506         int r, i;
 507
 508         if (!kiq->pmf || !kiq->pmf->kiq_map_queues || !kiq->pmf->kiq_set_resources)
 509                 return -EINVAL;
 510
 511         for (i = 0; i < AMDGPU_MAX_COMPUTE_QUEUES; ++i) {
 512                 if (!test_bit(i, adev->gfx.mec.queue_bitmap))
 513                         continue;
 514
 515                 /* This situation may be hit in the future if a new HW
 516                  * generation exposes more than 64 queues. If so, the
 517                  * definition of queue_mask needs updating */
 518                 if (WARN_ON(i > (sizeof(queue_mask)*8))) {
 519                         DRM_ERROR("Invalid KCQ enabled: %d\n", i);
 520                         break;
 521                 }
 522
 523                 queue_mask |= (1ull << amdgpu_queue_mask_bit_to_set_resource_bit(adev, i));
 524         }
 525
 526         DRM_INFO("kiq ring mec %d pipe %d q %d\n", kiq_ring->me, kiq_ring->pipe,
 527                                                         kiq_ring->queue);
 528         spin_lock(&adev->gfx.kiq.ring_lock);
 529         r = amdgpu_ring_alloc(kiq_ring, kiq->pmf->map_queues_size *
 530                                         adev->gfx.num_compute_rings +
 531                                         kiq->pmf->set_resources_size);
 532         if (r) {
 533                 DRM_ERROR("Failed to lock KIQ (%d).\n", r);
 534                 spin_unlock(&adev->gfx.kiq.ring_lock);
 535                 return r;
 536         }
 537
 538         kiq->pmf->kiq_set_resources(kiq_ring, queue_mask);
 539         for (i = 0; i < adev->gfx.num_compute_rings; i++)
 540                 kiq->pmf->kiq_map_queues(kiq_ring, &adev->gfx.compute_ring[i]);
 541
 542         r = amdgpu_ring_test_helper(kiq_ring);
 543         spin_unlock(&adev->gfx.kiq.ring_lock);
 544         if (r)
 545                 DRM_ERROR("KCQ enable failed\n");
 546
 547         return r;
 548 }
 549
 550 /* amdgpu_gfx_off_ctrl - Handle gfx off feature enable/disable
 551  *
 552  * @adev: amdgpu_device pointer
 553  * @bool enable true: enable gfx off feature, false: disable gfx off feature
 554  *
 555  * 1. gfx off feature will be enabled by gfx ip after gfx cg gp enabled.
 556  * 2. other client can send request to disable gfx off feature, the request should be honored.
 557  * 3. other client can cancel their request of disable gfx off feature
 558  * 4. other client should not send request to enable gfx off feature before disable gfx off feature.
 559  */
 560
 561 void amdgpu_gfx_off_ctrl(struct amdgpu_device *adev, bool enable)
 562 {
 563         unsigned long delay = GFX_OFF_DELAY_ENABLE;
 564
 565         if (!(adev->pm.pp_feature & PP_GFXOFF_MASK))
 566                 return;
 567
 568         mutex_lock(&adev->gfx.gfx_off_mutex);
 569
 570         if (enable) {
 571                 /* If the count is already 0, it means there's an imbalance bug somewhere.
 572                  * Note that the bug may be in a different caller than the one which triggers the
 573                  * WARN_ON_ONCE.
 574                  */
 575                 if (WARN_ON_ONCE(adev->gfx.gfx_off_req_count == 0))
 576                         goto unlock;
 577
 578                 adev->gfx.gfx_off_req_count--;
 579
 580                 if (adev->gfx.gfx_off_req_count == 0 &&
 581                     !adev->gfx.gfx_off_state) {
 582                         /* If going to s2idle, no need to wait */
 583                         if (adev->in_s0ix)
 584                                 delay = GFX_OFF_NO_DELAY;
 585                         schedule_delayed_work(&adev->gfx.gfx_off_delay_work,
 586                                               delay);
 587                 }
 588         } else {
 589                 if (adev->gfx.gfx_off_req_count == 0) {
 590                         cancel_delayed_work_sync(&adev->gfx.gfx_off_delay_work);
 591
 592                         if (adev->gfx.gfx_off_state &&
 593                             !amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, false)) {
 594                                 adev->gfx.gfx_off_state = false;
 595
 596                                 if (adev->gfx.funcs->init_spm_golden) {
 597                                         dev_dbg(adev->dev,
 598                                                 "GFXOFF is disabled, re-init SPM golden settings\n");
 599                                         amdgpu_gfx_init_spm_golden(adev);
 600                                 }
 601                         }
 602                 }
 603
 604                 adev->gfx.gfx_off_req_count++;
 605         }
 606
 607 unlock:
 608         mutex_unlock(&adev->gfx.gfx_off_mutex);
 609 }
 610
 611 int amdgpu_get_gfx_off_status(struct amdgpu_device *adev, uint32_t *value)
 612 {
 613
 614         int r = 0;
 615
 616         mutex_lock(&adev->gfx.gfx_off_mutex);
 617
 618         r = smu_get_status_gfxoff(adev, value);
 619
 620         mutex_unlock(&adev->gfx.gfx_off_mutex);
 621
 622         return r;
 623 }
 624
 625 int amdgpu_gfx_ras_late_init(struct amdgpu_device *adev)
 626 {
 627         int r;
 628         struct ras_fs_if fs_info = {
 629                 .sysfs_name = "gfx_err_count",
 630         };
 631         struct ras_ih_if ih_info = {
 632                 .cb = amdgpu_gfx_process_ras_data_cb,
 633         };
 634
 635         if (!adev->gfx.ras_if) {
 636                 adev->gfx.ras_if = kmalloc(sizeof(struct ras_common_if), GFP_KERNEL);
 637                 if (!adev->gfx.ras_if)
 638                         return -ENOMEM;
 639                 adev->gfx.ras_if->block = AMDGPU_RAS_BLOCK__GFX;
 640                 adev->gfx.ras_if->type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE;
 641                 adev->gfx.ras_if->sub_block_index = 0;
 642         }
 643         fs_info.head = ih_info.head = *adev->gfx.ras_if;
 644         r = amdgpu_ras_late_init(adev, adev->gfx.ras_if,
 645                                  &fs_info, &ih_info);
 646         if (r)
 647                 goto free;
 648
 649         if (amdgpu_ras_is_supported(adev, adev->gfx.ras_if->block)) {
 650                 if (!amdgpu_persistent_edc_harvesting_supported(adev))
 651                         amdgpu_ras_reset_error_status(adev, AMDGPU_RAS_BLOCK__GFX);
 652
 653                 r = amdgpu_irq_get(adev, &adev->gfx.cp_ecc_error_irq, 0);
 654                 if (r)
 655                         goto late_fini;
 656         } else {
 657                 /* free gfx ras_if if ras is not supported */
 658                 r = 0;
 659                 goto free;
 660         }
 661
 662         return 0;
 663 late_fini:
 664         amdgpu_ras_late_fini(adev, adev->gfx.ras_if, &ih_info);
 665 free:
 666         kfree(adev->gfx.ras_if);
 667         adev->gfx.ras_if = NULL;
 668         return r;
 669 }
 670
 671 void amdgpu_gfx_ras_fini(struct amdgpu_device *adev)
 672 {
 673         if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX) &&
 674                         adev->gfx.ras_if) {
 675                 struct ras_common_if *ras_if = adev->gfx.ras_if;
 676                 struct ras_ih_if ih_info = {
 677                         .head = *ras_if,
 678                         .cb = amdgpu_gfx_process_ras_data_cb,
 679                 };
 680
 681                 amdgpu_ras_late_fini(adev, ras_if, &ih_info);
 682                 kfree(ras_if);
 683         }
 684 }
 685
 686 int amdgpu_gfx_process_ras_data_cb(struct amdgpu_device *adev,
 687                 void *err_data,
 688                 struct amdgpu_iv_entry *entry)
 689 {
 690         /* TODO ue will trigger an interrupt.
 691          *
 692          * When “Full RAS” is enabled, the per-IP interrupt sources should
 693          * be disabled and the driver should only look for the aggregated
 694          * interrupt via sync flood
 695          */
 696         if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) {
 697                 kgd2kfd_set_sram_ecc_flag(adev->kfd.dev);
 698                 if (adev->gfx.ras_funcs &&
 699                     adev->gfx.ras_funcs->query_ras_error_count)
 700                         adev->gfx.ras_funcs->query_ras_error_count(adev, err_data);
 701                 amdgpu_ras_reset_gpu(adev);
 702         }
 703         return AMDGPU_RAS_SUCCESS;
 704 }
 705
 706 int amdgpu_gfx_cp_ecc_error_irq(struct amdgpu_device *adev,
 707                                   struct amdgpu_irq_src *source,
 708                                   struct amdgpu_iv_entry *entry)
 709 {
 710         struct ras_common_if *ras_if = adev->gfx.ras_if;
 711         struct ras_dispatch_if ih_data = {
 712                 .entry = entry,
 713         };
 714
 715         if (!ras_if)
 716                 return 0;
 717
 718         ih_data.head = *ras_if;
 719
 720         DRM_ERROR("CP ECC ERROR IRQ\n");
 721         amdgpu_ras_interrupt_dispatch(adev, &ih_data);
 722         return 0;
 723 }
 724
 725 uint32_t amdgpu_kiq_rreg(struct amdgpu_device *adev, uint32_t reg)
 726 {
 727         signed long r, cnt = 0;
 728         unsigned long flags;
 729         uint32_t seq, reg_val_offs = 0, value = 0;
 730         struct amdgpu_kiq *kiq = &adev->gfx.kiq;
 731         struct amdgpu_ring *ring = &kiq->ring;
 732
 733         if (amdgpu_device_skip_hw_access(adev))
 734                 return 0;
 735
 736         BUG_ON(!ring->funcs->emit_rreg);
 737
 738         spin_lock_irqsave(&kiq->ring_lock, flags);
 739         if (amdgpu_device_wb_get(adev, &reg_val_offs)) {
 740                 pr_err("critical bug! too many kiq readers\n");
 741                 goto failed_unlock;
 742         }
 743         amdgpu_ring_alloc(ring, 32);
 744         amdgpu_ring_emit_rreg(ring, reg, reg_val_offs);
 745         r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT);
 746         if (r)
 747                 goto failed_undo;
 748
 749         amdgpu_ring_commit(ring);
 750         spin_unlock_irqrestore(&kiq->ring_lock, flags);
 751
 752         r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
 753
 754         /* don't wait anymore for gpu reset case because this way may
 755          * block gpu_recover() routine forever, e.g. this virt_kiq_rreg
 756          * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will
 757          * never return if we keep waiting in virt_kiq_rreg, which cause
 758          * gpu_recover() hang there.
 759          *
 760          * also don't wait anymore for IRQ context
 761          * */
 762         if (r < 1 && (amdgpu_in_reset(adev) || in_interrupt()))
 763                 goto failed_kiq_read;
 764
 765         might_sleep();
 766         while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) {
 767                 msleep(MAX_KIQ_REG_BAILOUT_INTERVAL);
 768                 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
 769         }
 770
 771         if (cnt > MAX_KIQ_REG_TRY)
 772                 goto failed_kiq_read;
 773
 774         mb();
 775         value = adev->wb.wb[reg_val_offs];
 776         amdgpu_device_wb_free(adev, reg_val_offs);
 777         return value;
 778
 779 failed_undo:
 780         amdgpu_ring_undo(ring);
 781 failed_unlock:
 782         spin_unlock_irqrestore(&kiq->ring_lock, flags);
 783 failed_kiq_read:
 784         if (reg_val_offs)
 785                 amdgpu_device_wb_free(adev, reg_val_offs);
 786         dev_err(adev->dev, "failed to read reg:%x\n", reg);
 787         return ~0;
 788 }
 789
 790 void amdgpu_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
 791 {
 792         signed long r, cnt = 0;
 793         unsigned long flags;
 794         uint32_t seq;
 795         struct amdgpu_kiq *kiq = &adev->gfx.kiq;
 796         struct amdgpu_ring *ring = &kiq->ring;
 797
 798         BUG_ON(!ring->funcs->emit_wreg);
 799
 800         if (amdgpu_device_skip_hw_access(adev))
 801                 return;
 802
 803         spin_lock_irqsave(&kiq->ring_lock, flags);
 804         amdgpu_ring_alloc(ring, 32);
 805         amdgpu_ring_emit_wreg(ring, reg, v);
 806         r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT);
 807         if (r)
 808                 goto failed_undo;
 809
 810         amdgpu_ring_commit(ring);
 811         spin_unlock_irqrestore(&kiq->ring_lock, flags);
 812
 813         r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
 814
 815         /* don't wait anymore for gpu reset case because this way may
 816          * block gpu_recover() routine forever, e.g. this virt_kiq_rreg
 817          * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will
 818          * never return if we keep waiting in virt_kiq_rreg, which cause
 819          * gpu_recover() hang there.
 820          *
 821          * also don't wait anymore for IRQ context
 822          * */
 823         if (r < 1 && (amdgpu_in_reset(adev) || in_interrupt()))
 824                 goto failed_kiq_write;
 825
 826         might_sleep();
 827         while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) {
 828
 829                 msleep(MAX_KIQ_REG_BAILOUT_INTERVAL);
 830                 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
 831         }
 832
 833         if (cnt > MAX_KIQ_REG_TRY)
 834                 goto failed_kiq_write;
 835
 836         return;
 837
 838 failed_undo:
 839         amdgpu_ring_undo(ring);
 840         spin_unlock_irqrestore(&kiq->ring_lock, flags);
 841 failed_kiq_write:
 842         dev_err(adev->dev, "failed to write reg:%x\n", reg);
 843 }
 844
 845 int amdgpu_gfx_get_num_kcq(struct amdgpu_device *adev)
 846 {
 847         if (amdgpu_num_kcq == -1) {
 848                 return 8;
 849         } else if (amdgpu_num_kcq > 8 || amdgpu_num_kcq < 0) {
 850                 dev_warn(adev->dev, "set kernel compute queue number to 8 due to invalid parameter provided by user\n");
 851                 return 8;
 852         }
 853         return amdgpu_num_kcq;
 854 }
 855
 856 /* amdgpu_gfx_state_change_set - Handle gfx power state change set
 857  * @adev: amdgpu_device pointer
 858  * @state: gfx power state(1 -sGpuChangeState_D0Entry and 2 -sGpuChangeState_D3Entry)
 859  *
 860  */
 861
 862 void amdgpu_gfx_state_change_set(struct amdgpu_device *adev, enum gfx_change_state state)
 863 {
 864         mutex_lock(&adev->pm.mutex);
 865         if (adev->powerplay.pp_funcs &&
 866             adev->powerplay.pp_funcs->gfx_state_change_set)
 867                 ((adev)->powerplay.pp_funcs->gfx_state_change_set(
 868                         (adev)->powerplay.pp_handle, state));
 869         mutex_unlock(&adev->pm.mutex);
 870 }