drivers/gpu/drm/i915/gt/intel_ring.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2019 Intel Corporation
   4  */
   5
   6 #include "gem/i915_gem_internal.h"
   7 #include "gem/i915_gem_lmem.h"
   8 #include "gem/i915_gem_object.h"
   9
  10 #include "i915_drv.h"
  11 #include "i915_vma.h"
  12 #include "intel_engine.h"
  13 #include "intel_engine_regs.h"
  14 #include "intel_gpu_commands.h"
  15 #include "intel_ring.h"
  16 #include "intel_timeline.h"
  17
  18 unsigned int intel_ring_update_space(struct intel_ring *ring)
  19 {
  20         unsigned int space;
  21
  22         space = __intel_ring_space(ring->head, ring->emit, ring->size);
  23
  24         ring->space = space;
  25         return space;
  26 }
  27
  28 void __intel_ring_pin(struct intel_ring *ring)
  29 {
  30         GEM_BUG_ON(!atomic_read(&ring->pin_count));
  31         atomic_inc(&ring->pin_count);
  32 }
  33
  34 int intel_ring_pin(struct intel_ring *ring, struct i915_gem_ww_ctx *ww)
  35 {
  36         struct i915_vma *vma = ring->vma;
  37         unsigned int flags;
  38         void *addr;
  39         int ret;
  40
  41         if (atomic_fetch_inc(&ring->pin_count))
  42                 return 0;
  43
  44         /* Ring wraparound at offset 0 sometimes hangs. No idea why. */
  45         flags = PIN_OFFSET_BIAS | i915_ggtt_pin_bias(vma);
  46
  47         if (i915_gem_object_is_stolen(vma->obj))
  48                 flags |= PIN_MAPPABLE;
  49         else
  50                 flags |= PIN_HIGH;
  51
  52         ret = i915_ggtt_pin(vma, ww, 0, flags);
  53         if (unlikely(ret))
  54                 goto err_unpin;
  55
  56         if (i915_vma_is_map_and_fenceable(vma)) {
  57                 addr = (void __force *)i915_vma_pin_iomap(vma);
  58         } else {
  59                 int type = i915_coherent_map_type(vma->vm->i915, vma->obj, false);
  60
  61                 addr = i915_gem_object_pin_map(vma->obj, type);
  62         }
  63
  64         if (IS_ERR(addr)) {
  65                 ret = PTR_ERR(addr);
  66                 goto err_ring;
  67         }
  68
  69         i915_vma_make_unshrinkable(vma);
  70
  71         /* Discard any unused bytes beyond that submitted to hw. */
  72         intel_ring_reset(ring, ring->emit);
  73
  74         ring->vaddr = addr;
  75         return 0;
  76
  77 err_ring:
  78         i915_vma_unpin(vma);
  79 err_unpin:
  80         atomic_dec(&ring->pin_count);
  81         return ret;
  82 }
  83
  84 void intel_ring_reset(struct intel_ring *ring, u32 tail)
  85 {
  86         tail = intel_ring_wrap(ring, tail);
  87         ring->tail = tail;
  88         ring->head = tail;
  89         ring->emit = tail;
  90         intel_ring_update_space(ring);
  91 }
  92
  93 void intel_ring_unpin(struct intel_ring *ring)
  94 {
  95         struct i915_vma *vma = ring->vma;
  96
  97         if (!atomic_dec_and_test(&ring->pin_count))
  98                 return;
  99
 100         i915_vma_unset_ggtt_write(vma);
 101         if (i915_vma_is_map_and_fenceable(vma))
 102                 i915_vma_unpin_iomap(vma);
 103         else
 104                 i915_gem_object_unpin_map(vma->obj);
 105
 106         i915_vma_make_purgeable(vma);
 107         i915_vma_unpin(vma);
 108 }
 109
 110 static struct i915_vma *create_ring_vma(struct i915_ggtt *ggtt, int size)
 111 {
 112         struct i915_address_space *vm = &ggtt->vm;
 113         struct drm_i915_private *i915 = vm->i915;
 114         struct drm_i915_gem_object *obj;
 115         struct i915_vma *vma;
 116
 117         obj = i915_gem_object_create_lmem(i915, size, I915_BO_ALLOC_VOLATILE |
 118                                           I915_BO_ALLOC_PM_VOLATILE);
 119         if (IS_ERR(obj) && i915_ggtt_has_aperture(ggtt))
 120                 obj = i915_gem_object_create_stolen(i915, size);
 121         if (IS_ERR(obj))
 122                 obj = i915_gem_object_create_internal(i915, size);
 123         if (IS_ERR(obj))
 124                 return ERR_CAST(obj);
 125
 126         /*
 127          * Mark ring buffers as read-only from GPU side (so no stray overwrites)
 128          * if supported by the platform's GGTT.
 129          */
 130         if (vm->has_read_only)
 131                 i915_gem_object_set_readonly(obj);
 132
 133         vma = i915_vma_instance(obj, vm, NULL);
 134         if (IS_ERR(vma))
 135                 goto err;
 136
 137         return vma;
 138
 139 err:
 140         i915_gem_object_put(obj);
 141         return vma;
 142 }
 143
 144 struct intel_ring *
 145 intel_engine_create_ring(struct intel_engine_cs *engine, int size)
 146 {
 147         struct drm_i915_private *i915 = engine->i915;
 148         struct intel_ring *ring;
 149         struct i915_vma *vma;
 150
 151         GEM_BUG_ON(!is_power_of_2(size));
 152         GEM_BUG_ON(RING_CTL_SIZE(size) & ~RING_NR_PAGES);
 153
 154         ring = kzalloc(sizeof(*ring), GFP_KERNEL);
 155         if (!ring)
 156                 return ERR_PTR(-ENOMEM);
 157
 158         kref_init(&ring->ref);
 159         ring->size = size;
 160         ring->wrap = BITS_PER_TYPE(ring->size) - ilog2(size);
 161
 162         /*
 163          * Workaround an erratum on the i830 which causes a hang if
 164          * the TAIL pointer points to within the last 2 cachelines
 165          * of the buffer.
 166          */
 167         ring->effective_size = size;
 168         if (IS_I830(i915) || IS_I845G(i915))
 169                 ring->effective_size -= 2 * CACHELINE_BYTES;
 170
 171         intel_ring_update_space(ring);
 172
 173         vma = create_ring_vma(engine->gt->ggtt, size);
 174         if (IS_ERR(vma)) {
 175                 kfree(ring);
 176                 return ERR_CAST(vma);
 177         }
 178         ring->vma = vma;
 179
 180         return ring;
 181 }
 182
 183 void intel_ring_free(struct kref *ref)
 184 {
 185         struct intel_ring *ring = container_of(ref, typeof(*ring), ref);
 186
 187         i915_vma_put(ring->vma);
 188         kfree(ring);
 189 }
 190
 191 static noinline int
 192 wait_for_space(struct intel_ring *ring,
 193                struct intel_timeline *tl,
 194                unsigned int bytes)
 195 {
 196         struct i915_request *target;
 197         long timeout;
 198
 199         if (intel_ring_update_space(ring) >= bytes)
 200                 return 0;
 201
 202         GEM_BUG_ON(list_empty(&tl->requests));
 203         list_for_each_entry(target, &tl->requests, link) {
 204                 if (target->ring != ring)
 205                         continue;
 206
 207                 /* Would completion of this request free enough space? */
 208                 if (bytes <= __intel_ring_space(target->postfix,
 209                                                 ring->emit, ring->size))
 210                         break;
 211         }
 212
 213         if (GEM_WARN_ON(&target->link == &tl->requests))
 214                 return -ENOSPC;
 215
 216         timeout = i915_request_wait(target,
 217                                     I915_WAIT_INTERRUPTIBLE,
 218                                     MAX_SCHEDULE_TIMEOUT);
 219         if (timeout < 0)
 220                 return timeout;
 221
 222         i915_request_retire_upto(target);
 223
 224         intel_ring_update_space(ring);
 225         GEM_BUG_ON(ring->space < bytes);
 226         return 0;
 227 }
 228
 229 u32 *intel_ring_begin(struct i915_request *rq, unsigned int num_dwords)
 230 {
 231         struct intel_ring *ring = rq->ring;
 232         const unsigned int remain_usable = ring->effective_size - ring->emit;
 233         const unsigned int bytes = num_dwords * sizeof(u32);
 234         unsigned int need_wrap = 0;
 235         unsigned int total_bytes;
 236         u32 *cs;
 237
 238         /* Packets must be qword aligned. */
 239         GEM_BUG_ON(num_dwords & 1);
 240
 241         total_bytes = bytes + rq->reserved_space;
 242         GEM_BUG_ON(total_bytes > ring->effective_size);
 243
 244         if (unlikely(total_bytes > remain_usable)) {
 245                 const int remain_actual = ring->size - ring->emit;
 246
 247                 if (bytes > remain_usable) {
 248                         /*
 249                          * Not enough space for the basic request. So need to
 250                          * flush out the remainder and then wait for
 251                          * base + reserved.
 252                          */
 253                         total_bytes += remain_actual;
 254                         need_wrap = remain_actual | 1;
 255                 } else  {
 256                         /*
 257                          * The base request will fit but the reserved space
 258                          * falls off the end. So we don't need an immediate
 259                          * wrap and only need to effectively wait for the
 260                          * reserved size from the start of ringbuffer.
 261                          */
 262                         total_bytes = rq->reserved_space + remain_actual;
 263                 }
 264         }
 265
 266         if (unlikely(total_bytes > ring->space)) {
 267                 int ret;
 268
 269                 /*
 270                  * Space is reserved in the ringbuffer for finalising the
 271                  * request, as that cannot be allowed to fail. During request
 272                  * finalisation, reserved_space is set to 0 to stop the
 273                  * overallocation and the assumption is that then we never need
 274                  * to wait (which has the risk of failing with EINTR).
 275                  *
 276                  * See also i915_request_alloc() and i915_request_add().
 277                  */
 278                 GEM_BUG_ON(!rq->reserved_space);
 279
 280                 ret = wait_for_space(ring,
 281                                      i915_request_timeline(rq),
 282                                      total_bytes);
 283                 if (unlikely(ret))
 284                         return ERR_PTR(ret);
 285         }
 286
 287         if (unlikely(need_wrap)) {
 288                 need_wrap &= ~1;
 289                 GEM_BUG_ON(need_wrap > ring->space);
 290                 GEM_BUG_ON(ring->emit + need_wrap > ring->size);
 291                 GEM_BUG_ON(!IS_ALIGNED(need_wrap, sizeof(u64)));
 292
 293                 /* Fill the tail with MI_NOOP */
 294                 memset64(ring->vaddr + ring->emit, 0, need_wrap / sizeof(u64));
 295                 ring->space -= need_wrap;
 296                 ring->emit = 0;
 297         }
 298
 299         GEM_BUG_ON(ring->emit > ring->size - bytes);
 300         GEM_BUG_ON(ring->space < bytes);
 301         cs = ring->vaddr + ring->emit;
 302         if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))
 303                 memset32(cs, POISON_INUSE, bytes / sizeof(*cs));
 304         ring->emit += bytes;
 305         ring->space -= bytes;
 306
 307         return cs;
 308 }
 309
 310 /* Align the ring tail to a cacheline boundary */
 311 int intel_ring_cacheline_align(struct i915_request *rq)
 312 {
 313         int num_dwords;
 314         void *cs;
 315
 316         num_dwords = (rq->ring->emit & (CACHELINE_BYTES - 1)) / sizeof(u32);
 317         if (num_dwords == 0)
 318                 return 0;
 319
 320         num_dwords = CACHELINE_DWORDS - num_dwords;
 321         GEM_BUG_ON(num_dwords & 1);
 322
 323         cs = intel_ring_begin(rq, num_dwords);
 324         if (IS_ERR(cs))
 325                 return PTR_ERR(cs);
 326
 327         memset64(cs, (u64)MI_NOOP << 32 | MI_NOOP, num_dwords / 2);
 328         intel_ring_advance(rq, cs + num_dwords);
 329
 330         GEM_BUG_ON(rq->ring->emit & (CACHELINE_BYTES - 1));
 331         return 0;
 332 }
 333
 334 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
 335 #include "selftest_ring.c"
 336 #endif