drivers/gpu/drm/i915/gt/intel_context.c

   1 /*
   2  * SPDX-License-Identifier: MIT
   3  *
   4  * Copyright © 2019 Intel Corporation
   5  */
   6
   7 #include "gem/i915_gem_context.h"
   8 #include "gem/i915_gem_pm.h"
   9
  10 #include "i915_drv.h"
  11 #include "i915_globals.h"
  12
  13 #include "intel_context.h"
  14 #include "intel_engine.h"
  15 #include "intel_engine_pm.h"
  16 #include "intel_ring.h"
  17
  18 static struct i915_global_context {
  19         struct i915_global base;
  20         struct kmem_cache *slab_ce;
  21 } global;
  22
  23 static struct intel_context *intel_context_alloc(void)
  24 {
  25         return kmem_cache_zalloc(global.slab_ce, GFP_KERNEL);
  26 }
  27
  28 void intel_context_free(struct intel_context *ce)
  29 {
  30         kmem_cache_free(global.slab_ce, ce);
  31 }
  32
  33 struct intel_context *
  34 intel_context_create(struct intel_engine_cs *engine)
  35 {
  36         struct intel_context *ce;
  37
  38         ce = intel_context_alloc();
  39         if (!ce)
  40                 return ERR_PTR(-ENOMEM);
  41
  42         intel_context_init(ce, engine);
  43         return ce;
  44 }
  45
  46 int intel_context_alloc_state(struct intel_context *ce)
  47 {
  48         int err = 0;
  49
  50         if (mutex_lock_interruptible(&ce->pin_mutex))
  51                 return -EINTR;
  52
  53         if (!test_bit(CONTEXT_ALLOC_BIT, &ce->flags)) {
  54                 if (intel_context_is_banned(ce)) {
  55                         err = -EIO;
  56                         goto unlock;
  57                 }
  58
  59                 err = ce->ops->alloc(ce);
  60                 if (unlikely(err))
  61                         goto unlock;
  62
  63                 set_bit(CONTEXT_ALLOC_BIT, &ce->flags);
  64         }
  65
  66 unlock:
  67         mutex_unlock(&ce->pin_mutex);
  68         return err;
  69 }
  70
  71 static int intel_context_active_acquire(struct intel_context *ce)
  72 {
  73         int err;
  74
  75         __i915_active_acquire(&ce->active);
  76
  77         if (intel_context_is_barrier(ce))
  78                 return 0;
  79
  80         /* Preallocate tracking nodes */
  81         err = i915_active_acquire_preallocate_barrier(&ce->active,
  82                                                       ce->engine);
  83         if (err)
  84                 i915_active_release(&ce->active);
  85
  86         return err;
  87 }
  88
  89 static void intel_context_active_release(struct intel_context *ce)
  90 {
  91         /* Nodes preallocated in intel_context_active() */
  92         i915_active_acquire_barrier(&ce->active);
  93         i915_active_release(&ce->active);
  94 }
  95
  96 int __intel_context_do_pin(struct intel_context *ce)
  97 {
  98         int err;
  99
 100         if (unlikely(!test_bit(CONTEXT_ALLOC_BIT, &ce->flags))) {
 101                 err = intel_context_alloc_state(ce);
 102                 if (err)
 103                         return err;
 104         }
 105
 106         err = i915_active_acquire(&ce->active);
 107         if (err)
 108                 return err;
 109
 110         if (mutex_lock_interruptible(&ce->pin_mutex)) {
 111                 err = -EINTR;
 112                 goto out_release;
 113         }
 114
 115         if (unlikely(intel_context_is_closed(ce))) {
 116                 err = -ENOENT;
 117                 goto out_unlock;
 118         }
 119
 120         if (likely(!atomic_add_unless(&ce->pin_count, 1, 0))) {
 121                 err = intel_context_active_acquire(ce);
 122                 if (unlikely(err))
 123                         goto out_unlock;
 124
 125                 err = ce->ops->pin(ce);
 126                 if (unlikely(err))
 127                         goto err_active;
 128
 129                 CE_TRACE(ce, "pin ring:{start:%08x, head:%04x, tail:%04x}\n",
 130                          i915_ggtt_offset(ce->ring->vma),
 131                          ce->ring->head, ce->ring->tail);
 132
 133                 smp_mb__before_atomic(); /* flush pin before it is visible */
 134                 atomic_inc(&ce->pin_count);
 135         }
 136
 137         GEM_BUG_ON(!intel_context_is_pinned(ce)); /* no overflow! */
 138         GEM_BUG_ON(i915_active_is_idle(&ce->active));
 139         goto out_unlock;
 140
 141 err_active:
 142         intel_context_active_release(ce);
 143 out_unlock:
 144         mutex_unlock(&ce->pin_mutex);
 145 out_release:
 146         i915_active_release(&ce->active);
 147         return err;
 148 }
 149
 150 void intel_context_unpin(struct intel_context *ce)
 151 {
 152         if (!atomic_dec_and_test(&ce->pin_count))
 153                 return;
 154
 155         CE_TRACE(ce, "unpin\n");
 156         ce->ops->unpin(ce);
 157
 158         /*
 159          * Once released, we may asynchronously drop the active reference.
 160          * As that may be the only reference keeping the context alive,
 161          * take an extra now so that it is not freed before we finish
 162          * dereferencing it.
 163          */
 164         intel_context_get(ce);
 165         intel_context_active_release(ce);
 166         intel_context_put(ce);
 167 }
 168
 169 static int __context_pin_state(struct i915_vma *vma)
 170 {
 171         unsigned int bias = i915_ggtt_pin_bias(vma) | PIN_OFFSET_BIAS;
 172         int err;
 173
 174         err = i915_ggtt_pin(vma, 0, bias | PIN_HIGH);
 175         if (err)
 176                 return err;
 177
 178         err = i915_active_acquire(&vma->active);
 179         if (err)
 180                 goto err_unpin;
 181
 182         /*
 183          * And mark it as a globally pinned object to let the shrinker know
 184          * it cannot reclaim the object until we release it.
 185          */
 186         i915_vma_make_unshrinkable(vma);
 187         vma->obj->mm.dirty = true;
 188
 189         return 0;
 190
 191 err_unpin:
 192         i915_vma_unpin(vma);
 193         return err;
 194 }
 195
 196 static void __context_unpin_state(struct i915_vma *vma)
 197 {
 198         i915_vma_make_shrinkable(vma);
 199         i915_active_release(&vma->active);
 200         __i915_vma_unpin(vma);
 201 }
 202
 203 static int __ring_active(struct intel_ring *ring)
 204 {
 205         int err;
 206
 207         err = intel_ring_pin(ring);
 208         if (err)
 209                 return err;
 210
 211         err = i915_active_acquire(&ring->vma->active);
 212         if (err)
 213                 goto err_pin;
 214
 215         return 0;
 216
 217 err_pin:
 218         intel_ring_unpin(ring);
 219         return err;
 220 }
 221
 222 static void __ring_retire(struct intel_ring *ring)
 223 {
 224         i915_active_release(&ring->vma->active);
 225         intel_ring_unpin(ring);
 226 }
 227
 228 __i915_active_call
 229 static void __intel_context_retire(struct i915_active *active)
 230 {
 231         struct intel_context *ce = container_of(active, typeof(*ce), active);
 232
 233         CE_TRACE(ce, "retire runtime: { total:%lluns, avg:%lluns }\n",
 234                  intel_context_get_total_runtime_ns(ce),
 235                  intel_context_get_avg_runtime_ns(ce));
 236
 237         set_bit(CONTEXT_VALID_BIT, &ce->flags);
 238         if (ce->state)
 239                 __context_unpin_state(ce->state);
 240
 241         intel_timeline_unpin(ce->timeline);
 242         __ring_retire(ce->ring);
 243
 244         intel_context_put(ce);
 245 }
 246
 247 static int __intel_context_active(struct i915_active *active)
 248 {
 249         struct intel_context *ce = container_of(active, typeof(*ce), active);
 250         int err;
 251
 252         CE_TRACE(ce, "active\n");
 253
 254         intel_context_get(ce);
 255
 256         err = __ring_active(ce->ring);
 257         if (err)
 258                 goto err_put;
 259
 260         err = intel_timeline_pin(ce->timeline);
 261         if (err)
 262                 goto err_ring;
 263
 264         if (!ce->state)
 265                 return 0;
 266
 267         err = __context_pin_state(ce->state);
 268         if (err)
 269                 goto err_timeline;
 270
 271         return 0;
 272
 273 err_timeline:
 274         intel_timeline_unpin(ce->timeline);
 275 err_ring:
 276         __ring_retire(ce->ring);
 277 err_put:
 278         intel_context_put(ce);
 279         return err;
 280 }
 281
 282 void
 283 intel_context_init(struct intel_context *ce,
 284                    struct intel_engine_cs *engine)
 285 {
 286         GEM_BUG_ON(!engine->cops);
 287         GEM_BUG_ON(!engine->gt->vm);
 288
 289         kref_init(&ce->ref);
 290
 291         ce->engine = engine;
 292         ce->ops = engine->cops;
 293         ce->sseu = engine->sseu;
 294         ce->ring = __intel_context_ring_size(SZ_4K);
 295
 296         ewma_runtime_init(&ce->runtime.avg);
 297
 298         ce->vm = i915_vm_get(engine->gt->vm);
 299
 300         INIT_LIST_HEAD(&ce->signal_link);
 301         INIT_LIST_HEAD(&ce->signals);
 302
 303         mutex_init(&ce->pin_mutex);
 304
 305         i915_active_init(&ce->active,
 306                          __intel_context_active, __intel_context_retire);
 307 }
 308
 309 void intel_context_fini(struct intel_context *ce)
 310 {
 311         if (ce->timeline)
 312                 intel_timeline_put(ce->timeline);
 313         i915_vm_put(ce->vm);
 314
 315         mutex_destroy(&ce->pin_mutex);
 316         i915_active_fini(&ce->active);
 317 }
 318
 319 static void i915_global_context_shrink(void)
 320 {
 321         kmem_cache_shrink(global.slab_ce);
 322 }
 323
 324 static void i915_global_context_exit(void)
 325 {
 326         kmem_cache_destroy(global.slab_ce);
 327 }
 328
 329 static struct i915_global_context global = { {
 330         .shrink = i915_global_context_shrink,
 331         .exit = i915_global_context_exit,
 332 } };
 333
 334 int __init i915_global_context_init(void)
 335 {
 336         global.slab_ce = KMEM_CACHE(intel_context, SLAB_HWCACHE_ALIGN);
 337         if (!global.slab_ce)
 338                 return -ENOMEM;
 339
 340         i915_global_register(&global.base);
 341         return 0;
 342 }
 343
 344 void intel_context_enter_engine(struct intel_context *ce)
 345 {
 346         intel_engine_pm_get(ce->engine);
 347         intel_timeline_enter(ce->timeline);
 348 }
 349
 350 void intel_context_exit_engine(struct intel_context *ce)
 351 {
 352         intel_timeline_exit(ce->timeline);
 353         intel_engine_pm_put(ce->engine);
 354 }
 355
 356 int intel_context_prepare_remote_request(struct intel_context *ce,
 357                                          struct i915_request *rq)
 358 {
 359         struct intel_timeline *tl = ce->timeline;
 360         int err;
 361
 362         /* Only suitable for use in remotely modifying this context */
 363         GEM_BUG_ON(rq->context == ce);
 364
 365         if (rcu_access_pointer(rq->timeline) != tl) { /* timeline sharing! */
 366                 /* Queue this switch after current activity by this context. */
 367                 err = i915_active_fence_set(&tl->last_request, rq);
 368                 if (err)
 369                         return err;
 370         }
 371
 372         /*
 373          * Guarantee context image and the timeline remains pinned until the
 374          * modifying request is retired by setting the ce activity tracker.
 375          *
 376          * But we only need to take one pin on the account of it. Or in other
 377          * words transfer the pinned ce object to tracked active request.
 378          */
 379         GEM_BUG_ON(i915_active_is_idle(&ce->active));
 380         return i915_active_add_request(&ce->active, rq);
 381 }
 382
 383 struct i915_request *intel_context_create_request(struct intel_context *ce)
 384 {
 385         struct i915_request *rq;
 386         int err;
 387
 388         err = intel_context_pin(ce);
 389         if (unlikely(err))
 390                 return ERR_PTR(err);
 391
 392         rq = i915_request_create(ce);
 393         intel_context_unpin(ce);
 394
 395         return rq;
 396 }
 397
 398 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
 399 #include "selftest_context.c"
 400 #endif