drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c

   1 /*
   2  * Copyright 2015 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: monk liu <[email protected]>
  23  */
  24
  25 #include <drm/drm_auth.h>
  26 #include <drm/drm_drv.h>
  27 #include "amdgpu.h"
  28 #include "amdgpu_sched.h"
  29 #include "amdgpu_ras.h"
  30 #include <linux/nospec.h>
  31
  32 #define to_amdgpu_ctx_entity(e) \
  33         container_of((e), struct amdgpu_ctx_entity, entity)
  34
  35 const unsigned int amdgpu_ctx_num_entities[AMDGPU_HW_IP_NUM] = {
  36         [AMDGPU_HW_IP_GFX]      =       1,
  37         [AMDGPU_HW_IP_COMPUTE]  =       4,
  38         [AMDGPU_HW_IP_DMA]      =       2,
  39         [AMDGPU_HW_IP_UVD]      =       1,
  40         [AMDGPU_HW_IP_VCE]      =       1,
  41         [AMDGPU_HW_IP_UVD_ENC]  =       1,
  42         [AMDGPU_HW_IP_VCN_DEC]  =       1,
  43         [AMDGPU_HW_IP_VCN_ENC]  =       1,
  44         [AMDGPU_HW_IP_VCN_JPEG] =       1,
  45 };
  46
  47 bool amdgpu_ctx_priority_is_valid(int32_t ctx_prio)
  48 {
  49         switch (ctx_prio) {
  50         case AMDGPU_CTX_PRIORITY_UNSET:
  51         case AMDGPU_CTX_PRIORITY_VERY_LOW:
  52         case AMDGPU_CTX_PRIORITY_LOW:
  53         case AMDGPU_CTX_PRIORITY_NORMAL:
  54         case AMDGPU_CTX_PRIORITY_HIGH:
  55         case AMDGPU_CTX_PRIORITY_VERY_HIGH:
  56                 return true;
  57         default:
  58                 return false;
  59         }
  60 }
  61
  62 static enum drm_sched_priority
  63 amdgpu_ctx_to_drm_sched_prio(int32_t ctx_prio)
  64 {
  65         switch (ctx_prio) {
  66         case AMDGPU_CTX_PRIORITY_UNSET:
  67                 return DRM_SCHED_PRIORITY_UNSET;
  68
  69         case AMDGPU_CTX_PRIORITY_VERY_LOW:
  70                 return DRM_SCHED_PRIORITY_MIN;
  71
  72         case AMDGPU_CTX_PRIORITY_LOW:
  73                 return DRM_SCHED_PRIORITY_MIN;
  74
  75         case AMDGPU_CTX_PRIORITY_NORMAL:
  76                 return DRM_SCHED_PRIORITY_NORMAL;
  77
  78         case AMDGPU_CTX_PRIORITY_HIGH:
  79                 return DRM_SCHED_PRIORITY_HIGH;
  80
  81         case AMDGPU_CTX_PRIORITY_VERY_HIGH:
  82                 return DRM_SCHED_PRIORITY_HIGH;
  83
  84         /* This should not happen as we sanitized userspace provided priority
  85          * already, WARN if this happens.
  86          */
  87         default:
  88                 WARN(1, "Invalid context priority %d\n", ctx_prio);
  89                 return DRM_SCHED_PRIORITY_NORMAL;
  90         }
  91
  92 }
  93
  94 static int amdgpu_ctx_priority_permit(struct drm_file *filp,
  95                                       int32_t priority)
  96 {
  97         if (!amdgpu_ctx_priority_is_valid(priority))
  98                 return -EINVAL;
  99
 100         /* NORMAL and below are accessible by everyone */
 101         if (priority <= AMDGPU_CTX_PRIORITY_NORMAL)
 102                 return 0;
 103
 104         if (capable(CAP_SYS_NICE))
 105                 return 0;
 106
 107         if (drm_is_current_master(filp))
 108                 return 0;
 109
 110         return -EACCES;
 111 }
 112
 113 static enum amdgpu_gfx_pipe_priority amdgpu_ctx_prio_to_gfx_pipe_prio(int32_t prio)
 114 {
 115         switch (prio) {
 116         case AMDGPU_CTX_PRIORITY_HIGH:
 117         case AMDGPU_CTX_PRIORITY_VERY_HIGH:
 118                 return AMDGPU_GFX_PIPE_PRIO_HIGH;
 119         default:
 120                 return AMDGPU_GFX_PIPE_PRIO_NORMAL;
 121         }
 122 }
 123
 124 static enum amdgpu_ring_priority_level amdgpu_ctx_sched_prio_to_ring_prio(int32_t prio)
 125 {
 126         switch (prio) {
 127         case AMDGPU_CTX_PRIORITY_HIGH:
 128                 return AMDGPU_RING_PRIO_1;
 129         case AMDGPU_CTX_PRIORITY_VERY_HIGH:
 130                 return AMDGPU_RING_PRIO_2;
 131         default:
 132                 return AMDGPU_RING_PRIO_0;
 133         }
 134 }
 135
 136 static unsigned int amdgpu_ctx_get_hw_prio(struct amdgpu_ctx *ctx, u32 hw_ip)
 137 {
 138         struct amdgpu_device *adev = ctx->mgr->adev;
 139         unsigned int hw_prio;
 140         int32_t ctx_prio;
 141
 142         ctx_prio = (ctx->override_priority == AMDGPU_CTX_PRIORITY_UNSET) ?
 143                         ctx->init_priority : ctx->override_priority;
 144
 145         switch (hw_ip) {
 146         case AMDGPU_HW_IP_GFX:
 147         case AMDGPU_HW_IP_COMPUTE:
 148                 hw_prio = amdgpu_ctx_prio_to_gfx_pipe_prio(ctx_prio);
 149                 break;
 150         case AMDGPU_HW_IP_VCE:
 151         case AMDGPU_HW_IP_VCN_ENC:
 152                 hw_prio = amdgpu_ctx_sched_prio_to_ring_prio(ctx_prio);
 153                 break;
 154         default:
 155                 hw_prio = AMDGPU_RING_PRIO_DEFAULT;
 156                 break;
 157         }
 158
 159         hw_ip = array_index_nospec(hw_ip, AMDGPU_HW_IP_NUM);
 160         if (adev->gpu_sched[hw_ip][hw_prio].num_scheds == 0)
 161                 hw_prio = AMDGPU_RING_PRIO_DEFAULT;
 162
 163         return hw_prio;
 164 }
 165
 166 /* Calculate the time spend on the hw */
 167 static ktime_t amdgpu_ctx_fence_time(struct dma_fence *fence)
 168 {
 169         struct drm_sched_fence *s_fence;
 170
 171         if (!fence)
 172                 return ns_to_ktime(0);
 173
 174         /* When the fence is not even scheduled it can't have spend time */
 175         s_fence = to_drm_sched_fence(fence);
 176         if (!test_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &s_fence->scheduled.flags))
 177                 return ns_to_ktime(0);
 178
 179         /* When it is still running account how much already spend */
 180         if (!test_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &s_fence->finished.flags))
 181                 return ktime_sub(ktime_get(), s_fence->scheduled.timestamp);
 182
 183         return ktime_sub(s_fence->finished.timestamp,
 184                          s_fence->scheduled.timestamp);
 185 }
 186
 187 static ktime_t amdgpu_ctx_entity_time(struct amdgpu_ctx *ctx,
 188                                       struct amdgpu_ctx_entity *centity)
 189 {
 190         ktime_t res = ns_to_ktime(0);
 191         uint32_t i;
 192
 193         spin_lock(&ctx->ring_lock);
 194         for (i = 0; i < amdgpu_sched_jobs; i++) {
 195                 res = ktime_add(res, amdgpu_ctx_fence_time(centity->fences[i]));
 196         }
 197         spin_unlock(&ctx->ring_lock);
 198         return res;
 199 }
 200
 201 static int amdgpu_ctx_init_entity(struct amdgpu_ctx *ctx, u32 hw_ip,
 202                                   const u32 ring)
 203 {
 204         struct drm_gpu_scheduler **scheds = NULL, *sched = NULL;
 205         struct amdgpu_device *adev = ctx->mgr->adev;
 206         struct amdgpu_ctx_entity *entity;
 207         enum drm_sched_priority drm_prio;
 208         unsigned int hw_prio, num_scheds;
 209         int32_t ctx_prio;
 210         int r;
 211
 212         entity = kzalloc(struct_size(entity, fences, amdgpu_sched_jobs),
 213                          GFP_KERNEL);
 214         if (!entity)
 215                 return  -ENOMEM;
 216
 217         ctx_prio = (ctx->override_priority == AMDGPU_CTX_PRIORITY_UNSET) ?
 218                         ctx->init_priority : ctx->override_priority;
 219         entity->hw_ip = hw_ip;
 220         entity->sequence = 1;
 221         hw_prio = amdgpu_ctx_get_hw_prio(ctx, hw_ip);
 222         drm_prio = amdgpu_ctx_to_drm_sched_prio(ctx_prio);
 223
 224         hw_ip = array_index_nospec(hw_ip, AMDGPU_HW_IP_NUM);
 225         scheds = adev->gpu_sched[hw_ip][hw_prio].sched;
 226         num_scheds = adev->gpu_sched[hw_ip][hw_prio].num_scheds;
 227
 228         /* disable load balance if the hw engine retains context among dependent jobs */
 229         if (hw_ip == AMDGPU_HW_IP_VCN_ENC ||
 230             hw_ip == AMDGPU_HW_IP_VCN_DEC ||
 231             hw_ip == AMDGPU_HW_IP_UVD_ENC ||
 232             hw_ip == AMDGPU_HW_IP_UVD) {
 233                 sched = drm_sched_pick_best(scheds, num_scheds);
 234                 scheds = &sched;
 235                 num_scheds = 1;
 236         }
 237
 238         r = drm_sched_entity_init(&entity->entity, drm_prio, scheds, num_scheds,
 239                                   &ctx->guilty);
 240         if (r)
 241                 goto error_free_entity;
 242
 243         /* It's not an error if we fail to install the new entity */
 244         if (cmpxchg(&ctx->entities[hw_ip][ring], NULL, entity))
 245                 goto cleanup_entity;
 246
 247         return 0;
 248
 249 cleanup_entity:
 250         drm_sched_entity_fini(&entity->entity);
 251
 252 error_free_entity:
 253         kfree(entity);
 254
 255         return r;
 256 }
 257
 258 static ktime_t amdgpu_ctx_fini_entity(struct amdgpu_ctx_entity *entity)
 259 {
 260         ktime_t res = ns_to_ktime(0);
 261         int i;
 262
 263         if (!entity)
 264                 return res;
 265
 266         for (i = 0; i < amdgpu_sched_jobs; ++i) {
 267                 res = ktime_add(res, amdgpu_ctx_fence_time(entity->fences[i]));
 268                 dma_fence_put(entity->fences[i]);
 269         }
 270
 271         kfree(entity);
 272         return res;
 273 }
 274
 275 static int amdgpu_ctx_init(struct amdgpu_ctx_mgr *mgr, int32_t priority,
 276                            struct drm_file *filp, struct amdgpu_ctx *ctx)
 277 {
 278         int r;
 279
 280         r = amdgpu_ctx_priority_permit(filp, priority);
 281         if (r)
 282                 return r;
 283
 284         memset(ctx, 0, sizeof(*ctx));
 285
 286         kref_init(&ctx->refcount);
 287         ctx->mgr = mgr;
 288         spin_lock_init(&ctx->ring_lock);
 289         mutex_init(&ctx->lock);
 290
 291         ctx->reset_counter = atomic_read(&mgr->adev->gpu_reset_counter);
 292         ctx->reset_counter_query = ctx->reset_counter;
 293         ctx->vram_lost_counter = atomic_read(&mgr->adev->vram_lost_counter);
 294         ctx->init_priority = priority;
 295         ctx->override_priority = AMDGPU_CTX_PRIORITY_UNSET;
 296         ctx->stable_pstate = AMDGPU_CTX_STABLE_PSTATE_NONE;
 297
 298         return 0;
 299 }
 300
 301 static int amdgpu_ctx_get_stable_pstate(struct amdgpu_ctx *ctx,
 302                                         u32 *stable_pstate)
 303 {
 304         struct amdgpu_device *adev = ctx->mgr->adev;
 305         enum amd_dpm_forced_level current_level;
 306
 307         current_level = amdgpu_dpm_get_performance_level(adev);
 308
 309         switch (current_level) {
 310         case AMD_DPM_FORCED_LEVEL_PROFILE_STANDARD:
 311                 *stable_pstate = AMDGPU_CTX_STABLE_PSTATE_STANDARD;
 312                 break;
 313         case AMD_DPM_FORCED_LEVEL_PROFILE_MIN_SCLK:
 314                 *stable_pstate = AMDGPU_CTX_STABLE_PSTATE_MIN_SCLK;
 315                 break;
 316         case AMD_DPM_FORCED_LEVEL_PROFILE_MIN_MCLK:
 317                 *stable_pstate = AMDGPU_CTX_STABLE_PSTATE_MIN_MCLK;
 318                 break;
 319         case AMD_DPM_FORCED_LEVEL_PROFILE_PEAK:
 320                 *stable_pstate = AMDGPU_CTX_STABLE_PSTATE_PEAK;
 321                 break;
 322         default:
 323                 *stable_pstate = AMDGPU_CTX_STABLE_PSTATE_NONE;
 324                 break;
 325         }
 326         return 0;
 327 }
 328
 329 static int amdgpu_ctx_set_stable_pstate(struct amdgpu_ctx *ctx,
 330                                         u32 stable_pstate)
 331 {
 332         struct amdgpu_device *adev = ctx->mgr->adev;
 333         enum amd_dpm_forced_level level;
 334         u32 current_stable_pstate;
 335         int r;
 336
 337         mutex_lock(&adev->pm.stable_pstate_ctx_lock);
 338         if (adev->pm.stable_pstate_ctx && adev->pm.stable_pstate_ctx != ctx) {
 339                 r = -EBUSY;
 340                 goto done;
 341         }
 342
 343         r = amdgpu_ctx_get_stable_pstate(ctx, &current_stable_pstate);
 344         if (r || (stable_pstate == current_stable_pstate))
 345                 goto done;
 346
 347         switch (stable_pstate) {
 348         case AMDGPU_CTX_STABLE_PSTATE_NONE:
 349                 level = AMD_DPM_FORCED_LEVEL_AUTO;
 350                 break;
 351         case AMDGPU_CTX_STABLE_PSTATE_STANDARD:
 352                 level = AMD_DPM_FORCED_LEVEL_PROFILE_STANDARD;
 353                 break;
 354         case AMDGPU_CTX_STABLE_PSTATE_MIN_SCLK:
 355                 level = AMD_DPM_FORCED_LEVEL_PROFILE_MIN_SCLK;
 356                 break;
 357         case AMDGPU_CTX_STABLE_PSTATE_MIN_MCLK:
 358                 level = AMD_DPM_FORCED_LEVEL_PROFILE_MIN_MCLK;
 359                 break;
 360         case AMDGPU_CTX_STABLE_PSTATE_PEAK:
 361                 level = AMD_DPM_FORCED_LEVEL_PROFILE_PEAK;
 362                 break;
 363         default:
 364                 r = -EINVAL;
 365                 goto done;
 366         }
 367
 368         r = amdgpu_dpm_force_performance_level(adev, level);
 369
 370         if (level == AMD_DPM_FORCED_LEVEL_AUTO)
 371                 adev->pm.stable_pstate_ctx = NULL;
 372         else
 373                 adev->pm.stable_pstate_ctx = ctx;
 374 done:
 375         mutex_unlock(&adev->pm.stable_pstate_ctx_lock);
 376
 377         return r;
 378 }
 379
 380 static void amdgpu_ctx_fini(struct kref *ref)
 381 {
 382         struct amdgpu_ctx *ctx = container_of(ref, struct amdgpu_ctx, refcount);
 383         struct amdgpu_ctx_mgr *mgr = ctx->mgr;
 384         struct amdgpu_device *adev = mgr->adev;
 385         unsigned i, j, idx;
 386
 387         if (!adev)
 388                 return;
 389
 390         for (i = 0; i < AMDGPU_HW_IP_NUM; ++i) {
 391                 for (j = 0; j < AMDGPU_MAX_ENTITY_NUM; ++j) {
 392                         ktime_t spend;
 393
 394                         spend = amdgpu_ctx_fini_entity(ctx->entities[i][j]);
 395                         atomic64_add(ktime_to_ns(spend), &mgr->time_spend[i]);
 396                 }
 397         }
 398
 399         if (drm_dev_enter(&adev->ddev, &idx)) {
 400                 amdgpu_ctx_set_stable_pstate(ctx, AMDGPU_CTX_STABLE_PSTATE_NONE);
 401                 drm_dev_exit(idx);
 402         }
 403
 404         mutex_destroy(&ctx->lock);
 405         kfree(ctx);
 406 }
 407
 408 int amdgpu_ctx_get_entity(struct amdgpu_ctx *ctx, u32 hw_ip, u32 instance,
 409                           u32 ring, struct drm_sched_entity **entity)
 410 {
 411         int r;
 412
 413         if (hw_ip >= AMDGPU_HW_IP_NUM) {
 414                 DRM_ERROR("unknown HW IP type: %d\n", hw_ip);
 415                 return -EINVAL;
 416         }
 417
 418         /* Right now all IPs have only one instance - multiple rings. */
 419         if (instance != 0) {
 420                 DRM_DEBUG("invalid ip instance: %d\n", instance);
 421                 return -EINVAL;
 422         }
 423
 424         if (ring >= amdgpu_ctx_num_entities[hw_ip]) {
 425                 DRM_DEBUG("invalid ring: %d %d\n", hw_ip, ring);
 426                 return -EINVAL;
 427         }
 428
 429         if (ctx->entities[hw_ip][ring] == NULL) {
 430                 r = amdgpu_ctx_init_entity(ctx, hw_ip, ring);
 431                 if (r)
 432                         return r;
 433         }
 434
 435         *entity = &ctx->entities[hw_ip][ring]->entity;
 436         return 0;
 437 }
 438
 439 static int amdgpu_ctx_alloc(struct amdgpu_device *adev,
 440                             struct amdgpu_fpriv *fpriv,
 441                             struct drm_file *filp,
 442                             int32_t priority,
 443                             uint32_t *id)
 444 {
 445         struct amdgpu_ctx_mgr *mgr = &fpriv->ctx_mgr;
 446         struct amdgpu_ctx *ctx;
 447         int r;
 448
 449         ctx = kmalloc(sizeof(*ctx), GFP_KERNEL);
 450         if (!ctx)
 451                 return -ENOMEM;
 452
 453         mutex_lock(&mgr->lock);
 454         r = idr_alloc(&mgr->ctx_handles, ctx, 1, AMDGPU_VM_MAX_NUM_CTX, GFP_KERNEL);
 455         if (r < 0) {
 456                 mutex_unlock(&mgr->lock);
 457                 kfree(ctx);
 458                 return r;
 459         }
 460
 461         *id = (uint32_t)r;
 462         r = amdgpu_ctx_init(mgr, priority, filp, ctx);
 463         if (r) {
 464                 idr_remove(&mgr->ctx_handles, *id);
 465                 *id = 0;
 466                 kfree(ctx);
 467         }
 468         mutex_unlock(&mgr->lock);
 469         return r;
 470 }
 471
 472 static void amdgpu_ctx_do_release(struct kref *ref)
 473 {
 474         struct amdgpu_ctx *ctx;
 475         u32 i, j;
 476
 477         ctx = container_of(ref, struct amdgpu_ctx, refcount);
 478         for (i = 0; i < AMDGPU_HW_IP_NUM; ++i) {
 479                 for (j = 0; j < amdgpu_ctx_num_entities[i]; ++j) {
 480                         if (!ctx->entities[i][j])
 481                                 continue;
 482
 483                         drm_sched_entity_destroy(&ctx->entities[i][j]->entity);
 484                 }
 485         }
 486
 487         amdgpu_ctx_fini(ref);
 488 }
 489
 490 static int amdgpu_ctx_free(struct amdgpu_fpriv *fpriv, uint32_t id)
 491 {
 492         struct amdgpu_ctx_mgr *mgr = &fpriv->ctx_mgr;
 493         struct amdgpu_ctx *ctx;
 494
 495         mutex_lock(&mgr->lock);
 496         ctx = idr_remove(&mgr->ctx_handles, id);
 497         if (ctx)
 498                 kref_put(&ctx->refcount, amdgpu_ctx_do_release);
 499         mutex_unlock(&mgr->lock);
 500         return ctx ? 0 : -EINVAL;
 501 }
 502
 503 static int amdgpu_ctx_query(struct amdgpu_device *adev,
 504                             struct amdgpu_fpriv *fpriv, uint32_t id,
 505                             union drm_amdgpu_ctx_out *out)
 506 {
 507         struct amdgpu_ctx *ctx;
 508         struct amdgpu_ctx_mgr *mgr;
 509         unsigned reset_counter;
 510
 511         if (!fpriv)
 512                 return -EINVAL;
 513
 514         mgr = &fpriv->ctx_mgr;
 515         mutex_lock(&mgr->lock);
 516         ctx = idr_find(&mgr->ctx_handles, id);
 517         if (!ctx) {
 518                 mutex_unlock(&mgr->lock);
 519                 return -EINVAL;
 520         }
 521
 522         /* TODO: these two are always zero */
 523         out->state.flags = 0x0;
 524         out->state.hangs = 0x0;
 525
 526         /* determine if a GPU reset has occured since the last call */
 527         reset_counter = atomic_read(&adev->gpu_reset_counter);
 528         /* TODO: this should ideally return NO, GUILTY, or INNOCENT. */
 529         if (ctx->reset_counter_query == reset_counter)
 530                 out->state.reset_status = AMDGPU_CTX_NO_RESET;
 531         else
 532                 out->state.reset_status = AMDGPU_CTX_UNKNOWN_RESET;
 533         ctx->reset_counter_query = reset_counter;
 534
 535         mutex_unlock(&mgr->lock);
 536         return 0;
 537 }
 538
 539 #define AMDGPU_RAS_COUNTE_DELAY_MS 3000
 540
 541 static int amdgpu_ctx_query2(struct amdgpu_device *adev,
 542                              struct amdgpu_fpriv *fpriv, uint32_t id,
 543                              union drm_amdgpu_ctx_out *out)
 544 {
 545         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
 546         struct amdgpu_ctx *ctx;
 547         struct amdgpu_ctx_mgr *mgr;
 548
 549         if (!fpriv)
 550                 return -EINVAL;
 551
 552         mgr = &fpriv->ctx_mgr;
 553         mutex_lock(&mgr->lock);
 554         ctx = idr_find(&mgr->ctx_handles, id);
 555         if (!ctx) {
 556                 mutex_unlock(&mgr->lock);
 557                 return -EINVAL;
 558         }
 559
 560         out->state.flags = 0x0;
 561         out->state.hangs = 0x0;
 562
 563         if (ctx->reset_counter != atomic_read(&adev->gpu_reset_counter))
 564                 out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_RESET;
 565
 566         if (ctx->vram_lost_counter != atomic_read(&adev->vram_lost_counter))
 567                 out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_VRAMLOST;
 568
 569         if (atomic_read(&ctx->guilty))
 570                 out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_GUILTY;
 571
 572         if (adev->ras_enabled && con) {
 573                 /* Return the cached values in O(1),
 574                  * and schedule delayed work to cache
 575                  * new vaues.
 576                  */
 577                 int ce_count, ue_count;
 578
 579                 ce_count = atomic_read(&con->ras_ce_count);
 580                 ue_count = atomic_read(&con->ras_ue_count);
 581
 582                 if (ce_count != ctx->ras_counter_ce) {
 583                         ctx->ras_counter_ce = ce_count;
 584                         out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_RAS_CE;
 585                 }
 586
 587                 if (ue_count != ctx->ras_counter_ue) {
 588                         ctx->ras_counter_ue = ue_count;
 589                         out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_RAS_UE;
 590                 }
 591
 592                 schedule_delayed_work(&con->ras_counte_delay_work,
 593                                       msecs_to_jiffies(AMDGPU_RAS_COUNTE_DELAY_MS));
 594         }
 595
 596         mutex_unlock(&mgr->lock);
 597         return 0;
 598 }
 599
 600
 601
 602 static int amdgpu_ctx_stable_pstate(struct amdgpu_device *adev,
 603                                     struct amdgpu_fpriv *fpriv, uint32_t id,
 604                                     bool set, u32 *stable_pstate)
 605 {
 606         struct amdgpu_ctx *ctx;
 607         struct amdgpu_ctx_mgr *mgr;
 608         int r;
 609
 610         if (!fpriv)
 611                 return -EINVAL;
 612
 613         mgr = &fpriv->ctx_mgr;
 614         mutex_lock(&mgr->lock);
 615         ctx = idr_find(&mgr->ctx_handles, id);
 616         if (!ctx) {
 617                 mutex_unlock(&mgr->lock);
 618                 return -EINVAL;
 619         }
 620
 621         if (set)
 622                 r = amdgpu_ctx_set_stable_pstate(ctx, *stable_pstate);
 623         else
 624                 r = amdgpu_ctx_get_stable_pstate(ctx, stable_pstate);
 625
 626         mutex_unlock(&mgr->lock);
 627         return r;
 628 }
 629
 630 int amdgpu_ctx_ioctl(struct drm_device *dev, void *data,
 631                      struct drm_file *filp)
 632 {
 633         int r;
 634         uint32_t id, stable_pstate;
 635         int32_t priority;
 636
 637         union drm_amdgpu_ctx *args = data;
 638         struct amdgpu_device *adev = drm_to_adev(dev);
 639         struct amdgpu_fpriv *fpriv = filp->driver_priv;
 640
 641         id = args->in.ctx_id;
 642         priority = args->in.priority;
 643
 644         /* For backwards compatibility reasons, we need to accept
 645          * ioctls with garbage in the priority field */
 646         if (!amdgpu_ctx_priority_is_valid(priority))
 647                 priority = AMDGPU_CTX_PRIORITY_NORMAL;
 648
 649         switch (args->in.op) {
 650         case AMDGPU_CTX_OP_ALLOC_CTX:
 651                 r = amdgpu_ctx_alloc(adev, fpriv, filp, priority, &id);
 652                 args->out.alloc.ctx_id = id;
 653                 break;
 654         case AMDGPU_CTX_OP_FREE_CTX:
 655                 r = amdgpu_ctx_free(fpriv, id);
 656                 break;
 657         case AMDGPU_CTX_OP_QUERY_STATE:
 658                 r = amdgpu_ctx_query(adev, fpriv, id, &args->out);
 659                 break;
 660         case AMDGPU_CTX_OP_QUERY_STATE2:
 661                 r = amdgpu_ctx_query2(adev, fpriv, id, &args->out);
 662                 break;
 663         case AMDGPU_CTX_OP_GET_STABLE_PSTATE:
 664                 if (args->in.flags)
 665                         return -EINVAL;
 666                 r = amdgpu_ctx_stable_pstate(adev, fpriv, id, false, &stable_pstate);
 667                 if (!r)
 668                         args->out.pstate.flags = stable_pstate;
 669                 break;
 670         case AMDGPU_CTX_OP_SET_STABLE_PSTATE:
 671                 if (args->in.flags & ~AMDGPU_CTX_STABLE_PSTATE_FLAGS_MASK)
 672                         return -EINVAL;
 673                 stable_pstate = args->in.flags & AMDGPU_CTX_STABLE_PSTATE_FLAGS_MASK;
 674                 if (stable_pstate > AMDGPU_CTX_STABLE_PSTATE_PEAK)
 675                         return -EINVAL;
 676                 r = amdgpu_ctx_stable_pstate(adev, fpriv, id, true, &stable_pstate);
 677                 break;
 678         default:
 679                 return -EINVAL;
 680         }
 681
 682         return r;
 683 }
 684
 685 struct amdgpu_ctx *amdgpu_ctx_get(struct amdgpu_fpriv *fpriv, uint32_t id)
 686 {
 687         struct amdgpu_ctx *ctx;
 688         struct amdgpu_ctx_mgr *mgr;
 689
 690         if (!fpriv)
 691                 return NULL;
 692
 693         mgr = &fpriv->ctx_mgr;
 694
 695         mutex_lock(&mgr->lock);
 696         ctx = idr_find(&mgr->ctx_handles, id);
 697         if (ctx)
 698                 kref_get(&ctx->refcount);
 699         mutex_unlock(&mgr->lock);
 700         return ctx;
 701 }
 702
 703 int amdgpu_ctx_put(struct amdgpu_ctx *ctx)
 704 {
 705         if (ctx == NULL)
 706                 return -EINVAL;
 707
 708         kref_put(&ctx->refcount, amdgpu_ctx_do_release);
 709         return 0;
 710 }
 711
 712 uint64_t amdgpu_ctx_add_fence(struct amdgpu_ctx *ctx,
 713                               struct drm_sched_entity *entity,
 714                               struct dma_fence *fence)
 715 {
 716         struct amdgpu_ctx_entity *centity = to_amdgpu_ctx_entity(entity);
 717         uint64_t seq = centity->sequence;
 718         struct dma_fence *other = NULL;
 719         unsigned idx = 0;
 720
 721         idx = seq & (amdgpu_sched_jobs - 1);
 722         other = centity->fences[idx];
 723         WARN_ON(other && !dma_fence_is_signaled(other));
 724
 725         dma_fence_get(fence);
 726
 727         spin_lock(&ctx->ring_lock);
 728         centity->fences[idx] = fence;
 729         centity->sequence++;
 730         spin_unlock(&ctx->ring_lock);
 731
 732         atomic64_add(ktime_to_ns(amdgpu_ctx_fence_time(other)),
 733                      &ctx->mgr->time_spend[centity->hw_ip]);
 734
 735         dma_fence_put(other);
 736         return seq;
 737 }
 738
 739 struct dma_fence *amdgpu_ctx_get_fence(struct amdgpu_ctx *ctx,
 740                                        struct drm_sched_entity *entity,
 741                                        uint64_t seq)
 742 {
 743         struct amdgpu_ctx_entity *centity = to_amdgpu_ctx_entity(entity);
 744         struct dma_fence *fence;
 745
 746         spin_lock(&ctx->ring_lock);
 747
 748         if (seq == ~0ull)
 749                 seq = centity->sequence - 1;
 750
 751         if (seq >= centity->sequence) {
 752                 spin_unlock(&ctx->ring_lock);
 753                 return ERR_PTR(-EINVAL);
 754         }
 755
 756
 757         if (seq + amdgpu_sched_jobs < centity->sequence) {
 758                 spin_unlock(&ctx->ring_lock);
 759                 return NULL;
 760         }
 761
 762         fence = dma_fence_get(centity->fences[seq & (amdgpu_sched_jobs - 1)]);
 763         spin_unlock(&ctx->ring_lock);
 764
 765         return fence;
 766 }
 767
 768 static void amdgpu_ctx_set_entity_priority(struct amdgpu_ctx *ctx,
 769                                            struct amdgpu_ctx_entity *aentity,
 770                                            int hw_ip,
 771                                            int32_t priority)
 772 {
 773         struct amdgpu_device *adev = ctx->mgr->adev;
 774         unsigned int hw_prio;
 775         struct drm_gpu_scheduler **scheds = NULL;
 776         unsigned num_scheds;
 777
 778         /* set sw priority */
 779         drm_sched_entity_set_priority(&aentity->entity,
 780                                       amdgpu_ctx_to_drm_sched_prio(priority));
 781
 782         /* set hw priority */
 783         if (hw_ip == AMDGPU_HW_IP_COMPUTE || hw_ip == AMDGPU_HW_IP_GFX) {
 784                 hw_prio = amdgpu_ctx_get_hw_prio(ctx, hw_ip);
 785                 hw_prio = array_index_nospec(hw_prio, AMDGPU_RING_PRIO_MAX);
 786                 scheds = adev->gpu_sched[hw_ip][hw_prio].sched;
 787                 num_scheds = adev->gpu_sched[hw_ip][hw_prio].num_scheds;
 788                 drm_sched_entity_modify_sched(&aentity->entity, scheds,
 789                                               num_scheds);
 790         }
 791 }
 792
 793 void amdgpu_ctx_priority_override(struct amdgpu_ctx *ctx,
 794                                   int32_t priority)
 795 {
 796         int32_t ctx_prio;
 797         unsigned i, j;
 798
 799         ctx->override_priority = priority;
 800
 801         ctx_prio = (ctx->override_priority == AMDGPU_CTX_PRIORITY_UNSET) ?
 802                         ctx->init_priority : ctx->override_priority;
 803         for (i = 0; i < AMDGPU_HW_IP_NUM; ++i) {
 804                 for (j = 0; j < amdgpu_ctx_num_entities[i]; ++j) {
 805                         if (!ctx->entities[i][j])
 806                                 continue;
 807
 808                         amdgpu_ctx_set_entity_priority(ctx, ctx->entities[i][j],
 809                                                        i, ctx_prio);
 810                 }
 811         }
 812 }
 813
 814 int amdgpu_ctx_wait_prev_fence(struct amdgpu_ctx *ctx,
 815                                struct drm_sched_entity *entity)
 816 {
 817         struct amdgpu_ctx_entity *centity = to_amdgpu_ctx_entity(entity);
 818         struct dma_fence *other;
 819         unsigned idx;
 820         long r;
 821
 822         spin_lock(&ctx->ring_lock);
 823         idx = centity->sequence & (amdgpu_sched_jobs - 1);
 824         other = dma_fence_get(centity->fences[idx]);
 825         spin_unlock(&ctx->ring_lock);
 826
 827         if (!other)
 828                 return 0;
 829
 830         r = dma_fence_wait(other, true);
 831         if (r < 0 && r != -ERESTARTSYS)
 832                 DRM_ERROR("Error (%ld) waiting for fence!\n", r);
 833
 834         dma_fence_put(other);
 835         return r;
 836 }
 837
 838 void amdgpu_ctx_mgr_init(struct amdgpu_ctx_mgr *mgr,
 839                          struct amdgpu_device *adev)
 840 {
 841         unsigned int i;
 842
 843         mgr->adev = adev;
 844         mutex_init(&mgr->lock);
 845         idr_init(&mgr->ctx_handles);
 846
 847         for (i = 0; i < AMDGPU_HW_IP_NUM; ++i)
 848                 atomic64_set(&mgr->time_spend[i], 0);
 849 }
 850
 851 long amdgpu_ctx_mgr_entity_flush(struct amdgpu_ctx_mgr *mgr, long timeout)
 852 {
 853         struct amdgpu_ctx *ctx;
 854         struct idr *idp;
 855         uint32_t id, i, j;
 856
 857         idp = &mgr->ctx_handles;
 858
 859         mutex_lock(&mgr->lock);
 860         idr_for_each_entry(idp, ctx, id) {
 861                 for (i = 0; i < AMDGPU_HW_IP_NUM; ++i) {
 862                         for (j = 0; j < amdgpu_ctx_num_entities[i]; ++j) {
 863                                 struct drm_sched_entity *entity;
 864
 865                                 if (!ctx->entities[i][j])
 866                                         continue;
 867
 868                                 entity = &ctx->entities[i][j]->entity;
 869                                 timeout = drm_sched_entity_flush(entity, timeout);
 870                         }
 871                 }
 872         }
 873         mutex_unlock(&mgr->lock);
 874         return timeout;
 875 }
 876
 877 void amdgpu_ctx_mgr_entity_fini(struct amdgpu_ctx_mgr *mgr)
 878 {
 879         struct amdgpu_ctx *ctx;
 880         struct idr *idp;
 881         uint32_t id, i, j;
 882
 883         idp = &mgr->ctx_handles;
 884
 885         idr_for_each_entry(idp, ctx, id) {
 886                 if (kref_read(&ctx->refcount) != 1) {
 887                         DRM_ERROR("ctx %p is still alive\n", ctx);
 888                         continue;
 889                 }
 890
 891                 for (i = 0; i < AMDGPU_HW_IP_NUM; ++i) {
 892                         for (j = 0; j < amdgpu_ctx_num_entities[i]; ++j) {
 893                                 struct drm_sched_entity *entity;
 894
 895                                 if (!ctx->entities[i][j])
 896                                         continue;
 897
 898                                 entity = &ctx->entities[i][j]->entity;
 899                                 drm_sched_entity_fini(entity);
 900                         }
 901                 }
 902         }
 903 }
 904
 905 void amdgpu_ctx_mgr_fini(struct amdgpu_ctx_mgr *mgr)
 906 {
 907         struct amdgpu_ctx *ctx;
 908         struct idr *idp;
 909         uint32_t id;
 910
 911         amdgpu_ctx_mgr_entity_fini(mgr);
 912
 913         idp = &mgr->ctx_handles;
 914
 915         idr_for_each_entry(idp, ctx, id) {
 916                 if (kref_put(&ctx->refcount, amdgpu_ctx_fini) != 1)
 917                         DRM_ERROR("ctx %p is still alive\n", ctx);
 918         }
 919
 920         idr_destroy(&mgr->ctx_handles);
 921         mutex_destroy(&mgr->lock);
 922 }
 923
 924 void amdgpu_ctx_mgr_usage(struct amdgpu_ctx_mgr *mgr,
 925                           ktime_t usage[AMDGPU_HW_IP_NUM])
 926 {
 927         struct amdgpu_ctx *ctx;
 928         unsigned int hw_ip, i;
 929         uint32_t id;
 930
 931         /*
 932          * This is a little bit racy because it can be that a ctx or a fence are
 933          * destroyed just in the moment we try to account them. But that is ok
 934          * since exactly that case is explicitely allowed by the interface.
 935          */
 936         mutex_lock(&mgr->lock);
 937         for (hw_ip = 0; hw_ip < AMDGPU_HW_IP_NUM; ++hw_ip) {
 938                 uint64_t ns = atomic64_read(&mgr->time_spend[hw_ip]);
 939
 940                 usage[hw_ip] = ns_to_ktime(ns);
 941         }
 942
 943         idr_for_each_entry(&mgr->ctx_handles, ctx, id) {
 944                 for (hw_ip = 0; hw_ip < AMDGPU_HW_IP_NUM; ++hw_ip) {
 945                         for (i = 0; i < amdgpu_ctx_num_entities[hw_ip]; ++i) {
 946                                 struct amdgpu_ctx_entity *centity;
 947                                 ktime_t spend;
 948
 949                                 centity = ctx->entities[hw_ip][i];
 950                                 if (!centity)
 951                                         continue;
 952                                 spend = amdgpu_ctx_entity_time(ctx, centity);
 953                                 usage[hw_ip] = ktime_add(usage[hw_ip], spend);
 954                         }
 955                 }
 956         }
 957         mutex_unlock(&mgr->lock);
 958 }