drivers/gpu/drm/i915/i915_gem.c

   1 /*
   2  * Copyright © 2008-2015 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <[email protected]>
  25  *
  26  */
  27
  28 #include <drm/drm_vma_manager.h>
  29 #include <linux/dma-fence-array.h>
  30 #include <linux/kthread.h>
  31 #include <linux/dma-resv.h>
  32 #include <linux/shmem_fs.h>
  33 #include <linux/slab.h>
  34 #include <linux/stop_machine.h>
  35 #include <linux/swap.h>
  36 #include <linux/pci.h>
  37 #include <linux/dma-buf.h>
  38 #include <linux/mman.h>
  39
  40 #include "display/intel_display.h"
  41 #include "display/intel_frontbuffer.h"
  42
  43 #include "gem/i915_gem_clflush.h"
  44 #include "gem/i915_gem_context.h"
  45 #include "gem/i915_gem_ioctls.h"
  46 #include "gem/i915_gem_mman.h"
  47 #include "gem/i915_gem_region.h"
  48 #include "gt/intel_engine_user.h"
  49 #include "gt/intel_gt.h"
  50 #include "gt/intel_gt_pm.h"
  51 #include "gt/intel_workarounds.h"
  52
  53 #include "i915_drv.h"
  54 #include "i915_trace.h"
  55 #include "i915_vgpu.h"
  56
  57 #include "intel_pm.h"
  58
  59 static int
  60 insert_mappable_node(struct i915_ggtt *ggtt, struct drm_mm_node *node, u32 size)
  61 {
  62         int err;
  63
  64         err = mutex_lock_interruptible(&ggtt->vm.mutex);
  65         if (err)
  66                 return err;
  67
  68         memset(node, 0, sizeof(*node));
  69         err = drm_mm_insert_node_in_range(&ggtt->vm.mm, node,
  70                                           size, 0, I915_COLOR_UNEVICTABLE,
  71                                           0, ggtt->mappable_end,
  72                                           DRM_MM_INSERT_LOW);
  73
  74         mutex_unlock(&ggtt->vm.mutex);
  75
  76         return err;
  77 }
  78
  79 static void
  80 remove_mappable_node(struct i915_ggtt *ggtt, struct drm_mm_node *node)
  81 {
  82         mutex_lock(&ggtt->vm.mutex);
  83         drm_mm_remove_node(node);
  84         mutex_unlock(&ggtt->vm.mutex);
  85 }
  86
  87 int
  88 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
  89                             struct drm_file *file)
  90 {
  91         struct i915_ggtt *ggtt = &to_i915(dev)->ggtt;
  92         struct drm_i915_gem_get_aperture *args = data;
  93         struct i915_vma *vma;
  94         u64 pinned;
  95
  96         if (mutex_lock_interruptible(&ggtt->vm.mutex))
  97                 return -EINTR;
  98
  99         pinned = ggtt->vm.reserved;
 100         list_for_each_entry(vma, &ggtt->vm.bound_list, vm_link)
 101                 if (i915_vma_is_pinned(vma))
 102                         pinned += vma->node.size;
 103
 104         mutex_unlock(&ggtt->vm.mutex);
 105
 106         args->aper_size = ggtt->vm.total;
 107         args->aper_available_size = args->aper_size - pinned;
 108
 109         return 0;
 110 }
 111
 112 int i915_gem_object_unbind(struct drm_i915_gem_object *obj,
 113                            unsigned long flags)
 114 {
 115         struct intel_runtime_pm *rpm = &to_i915(obj->base.dev)->runtime_pm;
 116         LIST_HEAD(still_in_list);
 117         intel_wakeref_t wakeref;
 118         struct i915_vma *vma;
 119         int ret;
 120
 121         if (list_empty(&obj->vma.list))
 122                 return 0;
 123
 124         /*
 125          * As some machines use ACPI to handle runtime-resume callbacks, and
 126          * ACPI is quite kmalloc happy, we cannot resume beneath the vm->mutex
 127          * as they are required by the shrinker. Ergo, we wake the device up
 128          * first just in case.
 129          */
 130         wakeref = intel_runtime_pm_get(rpm);
 131
 132 try_again:
 133         ret = 0;
 134         spin_lock(&obj->vma.lock);
 135         while (!ret && (vma = list_first_entry_or_null(&obj->vma.list,
 136                                                        struct i915_vma,
 137                                                        obj_link))) {
 138                 struct i915_address_space *vm = vma->vm;
 139
 140                 list_move_tail(&vma->obj_link, &still_in_list);
 141                 if (!i915_vma_is_bound(vma, I915_VMA_BIND_MASK))
 142                         continue;
 143
 144                 if (flags & I915_GEM_OBJECT_UNBIND_TEST) {
 145                         ret = -EBUSY;
 146                         break;
 147                 }
 148
 149                 ret = -EAGAIN;
 150                 if (!i915_vm_tryopen(vm))
 151                         break;
 152
 153                 /* Prevent vma being freed by i915_vma_parked as we unbind */
 154                 vma = __i915_vma_get(vma);
 155                 spin_unlock(&obj->vma.lock);
 156
 157                 if (vma) {
 158                         ret = -EBUSY;
 159                         if (flags & I915_GEM_OBJECT_UNBIND_ACTIVE ||
 160                             !i915_vma_is_active(vma))
 161                                 ret = i915_vma_unbind(vma);
 162
 163                         __i915_vma_put(vma);
 164                 }
 165
 166                 i915_vm_close(vm);
 167                 spin_lock(&obj->vma.lock);
 168         }
 169         list_splice_init(&still_in_list, &obj->vma.list);
 170         spin_unlock(&obj->vma.lock);
 171
 172         if (ret == -EAGAIN && flags & I915_GEM_OBJECT_UNBIND_BARRIER) {
 173                 rcu_barrier(); /* flush the i915_vm_release() */
 174                 goto try_again;
 175         }
 176
 177         intel_runtime_pm_put(rpm, wakeref);
 178
 179         return ret;
 180 }
 181
 182 static int
 183 shmem_pread(struct page *page, int offset, int len, char __user *user_data,
 184             bool needs_clflush)
 185 {
 186         char *vaddr;
 187         int ret;
 188
 189         vaddr = kmap(page);
 190
 191         if (needs_clflush)
 192                 drm_clflush_virt_range(vaddr + offset, len);
 193
 194         ret = __copy_to_user(user_data, vaddr + offset, len);
 195
 196         kunmap(page);
 197
 198         return ret ? -EFAULT : 0;
 199 }
 200
 201 static int
 202 i915_gem_shmem_pread(struct drm_i915_gem_object *obj,
 203                      struct drm_i915_gem_pread *args)
 204 {
 205         unsigned int needs_clflush;
 206         unsigned int idx, offset;
 207         struct dma_fence *fence;
 208         char __user *user_data;
 209         u64 remain;
 210         int ret;
 211
 212         ret = i915_gem_object_lock_interruptible(obj, NULL);
 213         if (ret)
 214                 return ret;
 215
 216         ret = i915_gem_object_prepare_read(obj, &needs_clflush);
 217         if (ret) {
 218                 i915_gem_object_unlock(obj);
 219                 return ret;
 220         }
 221
 222         fence = i915_gem_object_lock_fence(obj);
 223         i915_gem_object_finish_access(obj);
 224         i915_gem_object_unlock(obj);
 225
 226         if (!fence)
 227                 return -ENOMEM;
 228
 229         remain = args->size;
 230         user_data = u64_to_user_ptr(args->data_ptr);
 231         offset = offset_in_page(args->offset);
 232         for (idx = args->offset >> PAGE_SHIFT; remain; idx++) {
 233                 struct page *page = i915_gem_object_get_page(obj, idx);
 234                 unsigned int length = min_t(u64, remain, PAGE_SIZE - offset);
 235
 236                 ret = shmem_pread(page, offset, length, user_data,
 237                                   needs_clflush);
 238                 if (ret)
 239                         break;
 240
 241                 remain -= length;
 242                 user_data += length;
 243                 offset = 0;
 244         }
 245
 246         i915_gem_object_unlock_fence(obj, fence);
 247         return ret;
 248 }
 249
 250 static inline bool
 251 gtt_user_read(struct io_mapping *mapping,
 252               loff_t base, int offset,
 253               char __user *user_data, int length)
 254 {
 255         void __iomem *vaddr;
 256         unsigned long unwritten;
 257
 258         /* We can use the cpu mem copy function because this is X86. */
 259         vaddr = io_mapping_map_atomic_wc(mapping, base);
 260         unwritten = __copy_to_user_inatomic(user_data,
 261                                             (void __force *)vaddr + offset,
 262                                             length);
 263         io_mapping_unmap_atomic(vaddr);
 264         if (unwritten) {
 265                 vaddr = io_mapping_map_wc(mapping, base, PAGE_SIZE);
 266                 unwritten = copy_to_user(user_data,
 267                                          (void __force *)vaddr + offset,
 268                                          length);
 269                 io_mapping_unmap(vaddr);
 270         }
 271         return unwritten;
 272 }
 273
 274 static int
 275 i915_gem_gtt_pread(struct drm_i915_gem_object *obj,
 276                    const struct drm_i915_gem_pread *args)
 277 {
 278         struct drm_i915_private *i915 = to_i915(obj->base.dev);
 279         struct i915_ggtt *ggtt = &i915->ggtt;
 280         intel_wakeref_t wakeref;
 281         struct drm_mm_node node;
 282         struct dma_fence *fence;
 283         void __user *user_data;
 284         struct i915_vma *vma;
 285         u64 remain, offset;
 286         int ret;
 287
 288         wakeref = intel_runtime_pm_get(&i915->runtime_pm);
 289         vma = ERR_PTR(-ENODEV);
 290         if (!i915_gem_object_is_tiled(obj))
 291                 vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0,
 292                                                PIN_MAPPABLE |
 293                                                PIN_NONBLOCK /* NOWARN */ |
 294                                                PIN_NOEVICT);
 295         if (!IS_ERR(vma)) {
 296                 node.start = i915_ggtt_offset(vma);
 297                 node.flags = 0;
 298         } else {
 299                 ret = insert_mappable_node(ggtt, &node, PAGE_SIZE);
 300                 if (ret)
 301                         goto out_rpm;
 302                 GEM_BUG_ON(!drm_mm_node_allocated(&node));
 303         }
 304
 305         ret = i915_gem_object_lock_interruptible(obj, NULL);
 306         if (ret)
 307                 goto out_unpin;
 308
 309         ret = i915_gem_object_set_to_gtt_domain(obj, false);
 310         if (ret) {
 311                 i915_gem_object_unlock(obj);
 312                 goto out_unpin;
 313         }
 314
 315         fence = i915_gem_object_lock_fence(obj);
 316         i915_gem_object_unlock(obj);
 317         if (!fence) {
 318                 ret = -ENOMEM;
 319                 goto out_unpin;
 320         }
 321
 322         user_data = u64_to_user_ptr(args->data_ptr);
 323         remain = args->size;
 324         offset = args->offset;
 325
 326         while (remain > 0) {
 327                 /* Operation in this page
 328                  *
 329                  * page_base = page offset within aperture
 330                  * page_offset = offset within page
 331                  * page_length = bytes to copy for this page
 332                  */
 333                 u32 page_base = node.start;
 334                 unsigned page_offset = offset_in_page(offset);
 335                 unsigned page_length = PAGE_SIZE - page_offset;
 336                 page_length = remain < page_length ? remain : page_length;
 337                 if (drm_mm_node_allocated(&node)) {
 338                         ggtt->vm.insert_page(&ggtt->vm,
 339                                              i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT),
 340                                              node.start, I915_CACHE_NONE, 0);
 341                 } else {
 342                         page_base += offset & PAGE_MASK;
 343                 }
 344
 345                 if (gtt_user_read(&ggtt->iomap, page_base, page_offset,
 346                                   user_data, page_length)) {
 347                         ret = -EFAULT;
 348                         break;
 349                 }
 350
 351                 remain -= page_length;
 352                 user_data += page_length;
 353                 offset += page_length;
 354         }
 355
 356         i915_gem_object_unlock_fence(obj, fence);
 357 out_unpin:
 358         if (drm_mm_node_allocated(&node)) {
 359                 ggtt->vm.clear_range(&ggtt->vm, node.start, node.size);
 360                 remove_mappable_node(ggtt, &node);
 361         } else {
 362                 i915_vma_unpin(vma);
 363         }
 364 out_rpm:
 365         intel_runtime_pm_put(&i915->runtime_pm, wakeref);
 366         return ret;
 367 }
 368
 369 /**
 370  * Reads data from the object referenced by handle.
 371  * @dev: drm device pointer
 372  * @data: ioctl data blob
 373  * @file: drm file pointer
 374  *
 375  * On error, the contents of *data are undefined.
 376  */
 377 int
 378 i915_gem_pread_ioctl(struct drm_device *dev, void *data,
 379                      struct drm_file *file)
 380 {
 381         struct drm_i915_gem_pread *args = data;
 382         struct drm_i915_gem_object *obj;
 383         int ret;
 384
 385         if (args->size == 0)
 386                 return 0;
 387
 388         if (!access_ok(u64_to_user_ptr(args->data_ptr),
 389                        args->size))
 390                 return -EFAULT;
 391
 392         obj = i915_gem_object_lookup(file, args->handle);
 393         if (!obj)
 394                 return -ENOENT;
 395
 396         /* Bounds check source.  */
 397         if (range_overflows_t(u64, args->offset, args->size, obj->base.size)) {
 398                 ret = -EINVAL;
 399                 goto out;
 400         }
 401
 402         trace_i915_gem_object_pread(obj, args->offset, args->size);
 403
 404         ret = -ENODEV;
 405         if (obj->ops->pread)
 406                 ret = obj->ops->pread(obj, args);
 407         if (ret != -ENODEV)
 408                 goto out;
 409
 410         ret = i915_gem_object_wait(obj,
 411                                    I915_WAIT_INTERRUPTIBLE,
 412                                    MAX_SCHEDULE_TIMEOUT);
 413         if (ret)
 414                 goto out;
 415
 416         ret = i915_gem_object_pin_pages(obj);
 417         if (ret)
 418                 goto out;
 419
 420         ret = i915_gem_shmem_pread(obj, args);
 421         if (ret == -EFAULT || ret == -ENODEV)
 422                 ret = i915_gem_gtt_pread(obj, args);
 423
 424         i915_gem_object_unpin_pages(obj);
 425 out:
 426         i915_gem_object_put(obj);
 427         return ret;
 428 }
 429
 430 /* This is the fast write path which cannot handle
 431  * page faults in the source data
 432  */
 433
 434 static inline bool
 435 ggtt_write(struct io_mapping *mapping,
 436            loff_t base, int offset,
 437            char __user *user_data, int length)
 438 {
 439         void __iomem *vaddr;
 440         unsigned long unwritten;
 441
 442         /* We can use the cpu mem copy function because this is X86. */
 443         vaddr = io_mapping_map_atomic_wc(mapping, base);
 444         unwritten = __copy_from_user_inatomic_nocache((void __force *)vaddr + offset,
 445                                                       user_data, length);
 446         io_mapping_unmap_atomic(vaddr);
 447         if (unwritten) {
 448                 vaddr = io_mapping_map_wc(mapping, base, PAGE_SIZE);
 449                 unwritten = copy_from_user((void __force *)vaddr + offset,
 450                                            user_data, length);
 451                 io_mapping_unmap(vaddr);
 452         }
 453
 454         return unwritten;
 455 }
 456
 457 /**
 458  * This is the fast pwrite path, where we copy the data directly from the
 459  * user into the GTT, uncached.
 460  * @obj: i915 GEM object
 461  * @args: pwrite arguments structure
 462  */
 463 static int
 464 i915_gem_gtt_pwrite_fast(struct drm_i915_gem_object *obj,
 465                          const struct drm_i915_gem_pwrite *args)
 466 {
 467         struct drm_i915_private *i915 = to_i915(obj->base.dev);
 468         struct i915_ggtt *ggtt = &i915->ggtt;
 469         struct intel_runtime_pm *rpm = &i915->runtime_pm;
 470         intel_wakeref_t wakeref;
 471         struct drm_mm_node node;
 472         struct dma_fence *fence;
 473         struct i915_vma *vma;
 474         u64 remain, offset;
 475         void __user *user_data;
 476         int ret;
 477
 478         if (i915_gem_object_has_struct_page(obj)) {
 479                 /*
 480                  * Avoid waking the device up if we can fallback, as
 481                  * waking/resuming is very slow (worst-case 10-100 ms
 482                  * depending on PCI sleeps and our own resume time).
 483                  * This easily dwarfs any performance advantage from
 484                  * using the cache bypass of indirect GGTT access.
 485                  */
 486                 wakeref = intel_runtime_pm_get_if_in_use(rpm);
 487                 if (!wakeref)
 488                         return -EFAULT;
 489         } else {
 490                 /* No backing pages, no fallback, we must force GGTT access */
 491                 wakeref = intel_runtime_pm_get(rpm);
 492         }
 493
 494         vma = ERR_PTR(-ENODEV);
 495         if (!i915_gem_object_is_tiled(obj))
 496                 vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0,
 497                                                PIN_MAPPABLE |
 498                                                PIN_NONBLOCK /* NOWARN */ |
 499                                                PIN_NOEVICT);
 500         if (!IS_ERR(vma)) {
 501                 node.start = i915_ggtt_offset(vma);
 502                 node.flags = 0;
 503         } else {
 504                 ret = insert_mappable_node(ggtt, &node, PAGE_SIZE);
 505                 if (ret)
 506                         goto out_rpm;
 507                 GEM_BUG_ON(!drm_mm_node_allocated(&node));
 508         }
 509
 510         ret = i915_gem_object_lock_interruptible(obj, NULL);
 511         if (ret)
 512                 goto out_unpin;
 513
 514         ret = i915_gem_object_set_to_gtt_domain(obj, true);
 515         if (ret) {
 516                 i915_gem_object_unlock(obj);
 517                 goto out_unpin;
 518         }
 519
 520         fence = i915_gem_object_lock_fence(obj);
 521         i915_gem_object_unlock(obj);
 522         if (!fence) {
 523                 ret = -ENOMEM;
 524                 goto out_unpin;
 525         }
 526
 527         i915_gem_object_invalidate_frontbuffer(obj, ORIGIN_CPU);
 528
 529         user_data = u64_to_user_ptr(args->data_ptr);
 530         offset = args->offset;
 531         remain = args->size;
 532         while (remain) {
 533                 /* Operation in this page
 534                  *
 535                  * page_base = page offset within aperture
 536                  * page_offset = offset within page
 537                  * page_length = bytes to copy for this page
 538                  */
 539                 u32 page_base = node.start;
 540                 unsigned int page_offset = offset_in_page(offset);
 541                 unsigned int page_length = PAGE_SIZE - page_offset;
 542                 page_length = remain < page_length ? remain : page_length;
 543                 if (drm_mm_node_allocated(&node)) {
 544                         /* flush the write before we modify the GGTT */
 545                         intel_gt_flush_ggtt_writes(ggtt->vm.gt);
 546                         ggtt->vm.insert_page(&ggtt->vm,
 547                                              i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT),
 548                                              node.start, I915_CACHE_NONE, 0);
 549                         wmb(); /* flush modifications to the GGTT (insert_page) */
 550                 } else {
 551                         page_base += offset & PAGE_MASK;
 552                 }
 553                 /* If we get a fault while copying data, then (presumably) our
 554                  * source page isn't available.  Return the error and we'll
 555                  * retry in the slow path.
 556                  * If the object is non-shmem backed, we retry again with the
 557                  * path that handles page fault.
 558                  */
 559                 if (ggtt_write(&ggtt->iomap, page_base, page_offset,
 560                                user_data, page_length)) {
 561                         ret = -EFAULT;
 562                         break;
 563                 }
 564
 565                 remain -= page_length;
 566                 user_data += page_length;
 567                 offset += page_length;
 568         }
 569
 570         intel_gt_flush_ggtt_writes(ggtt->vm.gt);
 571         i915_gem_object_flush_frontbuffer(obj, ORIGIN_CPU);
 572
 573         i915_gem_object_unlock_fence(obj, fence);
 574 out_unpin:
 575         if (drm_mm_node_allocated(&node)) {
 576                 ggtt->vm.clear_range(&ggtt->vm, node.start, node.size);
 577                 remove_mappable_node(ggtt, &node);
 578         } else {
 579                 i915_vma_unpin(vma);
 580         }
 581 out_rpm:
 582         intel_runtime_pm_put(rpm, wakeref);
 583         return ret;
 584 }
 585
 586 /* Per-page copy function for the shmem pwrite fastpath.
 587  * Flushes invalid cachelines before writing to the target if
 588  * needs_clflush_before is set and flushes out any written cachelines after
 589  * writing if needs_clflush is set.
 590  */
 591 static int
 592 shmem_pwrite(struct page *page, int offset, int len, char __user *user_data,
 593              bool needs_clflush_before,
 594              bool needs_clflush_after)
 595 {
 596         char *vaddr;
 597         int ret;
 598
 599         vaddr = kmap(page);
 600
 601         if (needs_clflush_before)
 602                 drm_clflush_virt_range(vaddr + offset, len);
 603
 604         ret = __copy_from_user(vaddr + offset, user_data, len);
 605         if (!ret && needs_clflush_after)
 606                 drm_clflush_virt_range(vaddr + offset, len);
 607
 608         kunmap(page);
 609
 610         return ret ? -EFAULT : 0;
 611 }
 612
 613 static int
 614 i915_gem_shmem_pwrite(struct drm_i915_gem_object *obj,
 615                       const struct drm_i915_gem_pwrite *args)
 616 {
 617         unsigned int partial_cacheline_write;
 618         unsigned int needs_clflush;
 619         unsigned int offset, idx;
 620         struct dma_fence *fence;
 621         void __user *user_data;
 622         u64 remain;
 623         int ret;
 624
 625         ret = i915_gem_object_lock_interruptible(obj, NULL);
 626         if (ret)
 627                 return ret;
 628
 629         ret = i915_gem_object_prepare_write(obj, &needs_clflush);
 630         if (ret) {
 631                 i915_gem_object_unlock(obj);
 632                 return ret;
 633         }
 634
 635         fence = i915_gem_object_lock_fence(obj);
 636         i915_gem_object_finish_access(obj);
 637         i915_gem_object_unlock(obj);
 638
 639         if (!fence)
 640                 return -ENOMEM;
 641
 642         /* If we don't overwrite a cacheline completely we need to be
 643          * careful to have up-to-date data by first clflushing. Don't
 644          * overcomplicate things and flush the entire patch.
 645          */
 646         partial_cacheline_write = 0;
 647         if (needs_clflush & CLFLUSH_BEFORE)
 648                 partial_cacheline_write = boot_cpu_data.x86_clflush_size - 1;
 649
 650         user_data = u64_to_user_ptr(args->data_ptr);
 651         remain = args->size;
 652         offset = offset_in_page(args->offset);
 653         for (idx = args->offset >> PAGE_SHIFT; remain; idx++) {
 654                 struct page *page = i915_gem_object_get_page(obj, idx);
 655                 unsigned int length = min_t(u64, remain, PAGE_SIZE - offset);
 656
 657                 ret = shmem_pwrite(page, offset, length, user_data,
 658                                    (offset | length) & partial_cacheline_write,
 659                                    needs_clflush & CLFLUSH_AFTER);
 660                 if (ret)
 661                         break;
 662
 663                 remain -= length;
 664                 user_data += length;
 665                 offset = 0;
 666         }
 667
 668         i915_gem_object_flush_frontbuffer(obj, ORIGIN_CPU);
 669         i915_gem_object_unlock_fence(obj, fence);
 670
 671         return ret;
 672 }
 673
 674 /**
 675  * Writes data to the object referenced by handle.
 676  * @dev: drm device
 677  * @data: ioctl data blob
 678  * @file: drm file
 679  *
 680  * On error, the contents of the buffer that were to be modified are undefined.
 681  */
 682 int
 683 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
 684                       struct drm_file *file)
 685 {
 686         struct drm_i915_gem_pwrite *args = data;
 687         struct drm_i915_gem_object *obj;
 688         int ret;
 689
 690         if (args->size == 0)
 691                 return 0;
 692
 693         if (!access_ok(u64_to_user_ptr(args->data_ptr), args->size))
 694                 return -EFAULT;
 695
 696         obj = i915_gem_object_lookup(file, args->handle);
 697         if (!obj)
 698                 return -ENOENT;
 699
 700         /* Bounds check destination. */
 701         if (range_overflows_t(u64, args->offset, args->size, obj->base.size)) {
 702                 ret = -EINVAL;
 703                 goto err;
 704         }
 705
 706         /* Writes not allowed into this read-only object */
 707         if (i915_gem_object_is_readonly(obj)) {
 708                 ret = -EINVAL;
 709                 goto err;
 710         }
 711
 712         trace_i915_gem_object_pwrite(obj, args->offset, args->size);
 713
 714         ret = -ENODEV;
 715         if (obj->ops->pwrite)
 716                 ret = obj->ops->pwrite(obj, args);
 717         if (ret != -ENODEV)
 718                 goto err;
 719
 720         ret = i915_gem_object_wait(obj,
 721                                    I915_WAIT_INTERRUPTIBLE |
 722                                    I915_WAIT_ALL,
 723                                    MAX_SCHEDULE_TIMEOUT);
 724         if (ret)
 725                 goto err;
 726
 727         ret = i915_gem_object_pin_pages(obj);
 728         if (ret)
 729                 goto err;
 730
 731         ret = -EFAULT;
 732         /* We can only do the GTT pwrite on untiled buffers, as otherwise
 733          * it would end up going through the fenced access, and we'll get
 734          * different detiling behavior between reading and writing.
 735          * pread/pwrite currently are reading and writing from the CPU
 736          * perspective, requiring manual detiling by the client.
 737          */
 738         if (!i915_gem_object_has_struct_page(obj) ||
 739             cpu_write_needs_clflush(obj))
 740                 /* Note that the gtt paths might fail with non-page-backed user
 741                  * pointers (e.g. gtt mappings when moving data between
 742                  * textures). Fallback to the shmem path in that case.
 743                  */
 744                 ret = i915_gem_gtt_pwrite_fast(obj, args);
 745
 746         if (ret == -EFAULT || ret == -ENOSPC) {
 747                 if (i915_gem_object_has_struct_page(obj))
 748                         ret = i915_gem_shmem_pwrite(obj, args);
 749         }
 750
 751         i915_gem_object_unpin_pages(obj);
 752 err:
 753         i915_gem_object_put(obj);
 754         return ret;
 755 }
 756
 757 /**
 758  * Called when user space has done writes to this buffer
 759  * @dev: drm device
 760  * @data: ioctl data blob
 761  * @file: drm file
 762  */
 763 int
 764 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
 765                          struct drm_file *file)
 766 {
 767         struct drm_i915_gem_sw_finish *args = data;
 768         struct drm_i915_gem_object *obj;
 769
 770         obj = i915_gem_object_lookup(file, args->handle);
 771         if (!obj)
 772                 return -ENOENT;
 773
 774         /*
 775          * Proxy objects are barred from CPU access, so there is no
 776          * need to ban sw_finish as it is a nop.
 777          */
 778
 779         /* Pinned buffers may be scanout, so flush the cache */
 780         i915_gem_object_flush_if_display(obj);
 781         i915_gem_object_put(obj);
 782
 783         return 0;
 784 }
 785
 786 void i915_gem_runtime_suspend(struct drm_i915_private *i915)
 787 {
 788         struct drm_i915_gem_object *obj, *on;
 789         int i;
 790
 791         /*
 792          * Only called during RPM suspend. All users of the userfault_list
 793          * must be holding an RPM wakeref to ensure that this can not
 794          * run concurrently with themselves (and use the struct_mutex for
 795          * protection between themselves).
 796          */
 797
 798         list_for_each_entry_safe(obj, on,
 799                                  &i915->ggtt.userfault_list, userfault_link)
 800                 __i915_gem_object_release_mmap_gtt(obj);
 801
 802         /*
 803          * The fence will be lost when the device powers down. If any were
 804          * in use by hardware (i.e. they are pinned), we should not be powering
 805          * down! All other fences will be reacquired by the user upon waking.
 806          */
 807         for (i = 0; i < i915->ggtt.num_fences; i++) {
 808                 struct i915_fence_reg *reg = &i915->ggtt.fence_regs[i];
 809
 810                 /*
 811                  * Ideally we want to assert that the fence register is not
 812                  * live at this point (i.e. that no piece of code will be
 813                  * trying to write through fence + GTT, as that both violates
 814                  * our tracking of activity and associated locking/barriers,
 815                  * but also is illegal given that the hw is powered down).
 816                  *
 817                  * Previously we used reg->pin_count as a "liveness" indicator.
 818                  * That is not sufficient, and we need a more fine-grained
 819                  * tool if we want to have a sanity check here.
 820                  */
 821
 822                 if (!reg->vma)
 823                         continue;
 824
 825                 GEM_BUG_ON(i915_vma_has_userfault(reg->vma));
 826                 reg->dirty = true;
 827         }
 828 }
 829
 830 static void discard_ggtt_vma(struct i915_vma *vma)
 831 {
 832         struct drm_i915_gem_object *obj = vma->obj;
 833
 834         spin_lock(&obj->vma.lock);
 835         if (!RB_EMPTY_NODE(&vma->obj_node)) {
 836                 rb_erase(&vma->obj_node, &obj->vma.tree);
 837                 RB_CLEAR_NODE(&vma->obj_node);
 838         }
 839         spin_unlock(&obj->vma.lock);
 840 }
 841
 842 struct i915_vma *
 843 i915_gem_object_ggtt_pin_ww(struct drm_i915_gem_object *obj,
 844                             struct i915_gem_ww_ctx *ww,
 845                             const struct i915_ggtt_view *view,
 846                             u64 size, u64 alignment, u64 flags)
 847 {
 848         struct drm_i915_private *i915 = to_i915(obj->base.dev);
 849         struct i915_ggtt *ggtt = &i915->ggtt;
 850         struct i915_vma *vma;
 851         int ret;
 852
 853         if (flags & PIN_MAPPABLE &&
 854             (!view || view->type == I915_GGTT_VIEW_NORMAL)) {
 855                 /*
 856                  * If the required space is larger than the available
 857                  * aperture, we will not able to find a slot for the
 858                  * object and unbinding the object now will be in
 859                  * vain. Worse, doing so may cause us to ping-pong
 860                  * the object in and out of the Global GTT and
 861                  * waste a lot of cycles under the mutex.
 862                  */
 863                 if (obj->base.size > ggtt->mappable_end)
 864                         return ERR_PTR(-E2BIG);
 865
 866                 /*
 867                  * If NONBLOCK is set the caller is optimistically
 868                  * trying to cache the full object within the mappable
 869                  * aperture, and *must* have a fallback in place for
 870                  * situations where we cannot bind the object. We
 871                  * can be a little more lax here and use the fallback
 872                  * more often to avoid costly migrations of ourselves
 873                  * and other objects within the aperture.
 874                  *
 875                  * Half-the-aperture is used as a simple heuristic.
 876                  * More interesting would to do search for a free
 877                  * block prior to making the commitment to unbind.
 878                  * That caters for the self-harm case, and with a
 879                  * little more heuristics (e.g. NOFAULT, NOEVICT)
 880                  * we could try to minimise harm to others.
 881                  */
 882                 if (flags & PIN_NONBLOCK &&
 883                     obj->base.size > ggtt->mappable_end / 2)
 884                         return ERR_PTR(-ENOSPC);
 885         }
 886
 887 new_vma:
 888         vma = i915_vma_instance(obj, &ggtt->vm, view);
 889         if (IS_ERR(vma))
 890                 return vma;
 891
 892         if (i915_vma_misplaced(vma, size, alignment, flags)) {
 893                 if (flags & PIN_NONBLOCK) {
 894                         if (i915_vma_is_pinned(vma) || i915_vma_is_active(vma))
 895                                 return ERR_PTR(-ENOSPC);
 896
 897                         if (flags & PIN_MAPPABLE &&
 898                             vma->fence_size > ggtt->mappable_end / 2)
 899                                 return ERR_PTR(-ENOSPC);
 900                 }
 901
 902                 if (i915_vma_is_pinned(vma) || i915_vma_is_active(vma)) {
 903                         discard_ggtt_vma(vma);
 904                         goto new_vma;
 905                 }
 906
 907                 ret = i915_vma_unbind(vma);
 908                 if (ret)
 909                         return ERR_PTR(ret);
 910         }
 911
 912         ret = i915_vma_pin_ww(vma, ww, size, alignment, flags | PIN_GLOBAL);
 913         if (ret)
 914                 return ERR_PTR(ret);
 915
 916         if (vma->fence && !i915_gem_object_is_tiled(obj)) {
 917                 mutex_lock(&ggtt->vm.mutex);
 918                 i915_vma_revoke_fence(vma);
 919                 mutex_unlock(&ggtt->vm.mutex);
 920         }
 921
 922         ret = i915_vma_wait_for_bind(vma);
 923         if (ret) {
 924                 i915_vma_unpin(vma);
 925                 return ERR_PTR(ret);
 926         }
 927
 928         return vma;
 929 }
 930
 931 int
 932 i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
 933                        struct drm_file *file_priv)
 934 {
 935         struct drm_i915_private *i915 = to_i915(dev);
 936         struct drm_i915_gem_madvise *args = data;
 937         struct drm_i915_gem_object *obj;
 938         int err;
 939
 940         switch (args->madv) {
 941         case I915_MADV_DONTNEED:
 942         case I915_MADV_WILLNEED:
 943             break;
 944         default:
 945             return -EINVAL;
 946         }
 947
 948         obj = i915_gem_object_lookup(file_priv, args->handle);
 949         if (!obj)
 950                 return -ENOENT;
 951
 952         err = mutex_lock_interruptible(&obj->mm.lock);
 953         if (err)
 954                 goto out;
 955
 956         if (i915_gem_object_has_pages(obj) &&
 957             i915_gem_object_is_tiled(obj) &&
 958             i915->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
 959                 if (obj->mm.madv == I915_MADV_WILLNEED) {
 960                         GEM_BUG_ON(!i915_gem_object_has_tiling_quirk(obj));
 961                         i915_gem_object_clear_tiling_quirk(obj);
 962                         i915_gem_object_make_shrinkable(obj);
 963                 }
 964                 if (args->madv == I915_MADV_WILLNEED) {
 965                         GEM_BUG_ON(i915_gem_object_has_tiling_quirk(obj));
 966                         i915_gem_object_make_unshrinkable(obj);
 967                         i915_gem_object_set_tiling_quirk(obj);
 968                 }
 969         }
 970
 971         if (obj->mm.madv != __I915_MADV_PURGED)
 972                 obj->mm.madv = args->madv;
 973
 974         if (i915_gem_object_has_pages(obj)) {
 975                 struct list_head *list;
 976
 977                 if (i915_gem_object_is_shrinkable(obj)) {
 978                         unsigned long flags;
 979
 980                         spin_lock_irqsave(&i915->mm.obj_lock, flags);
 981
 982                         if (obj->mm.madv != I915_MADV_WILLNEED)
 983                                 list = &i915->mm.purge_list;
 984                         else
 985                                 list = &i915->mm.shrink_list;
 986                         list_move_tail(&obj->mm.link, list);
 987
 988                         spin_unlock_irqrestore(&i915->mm.obj_lock, flags);
 989                 }
 990         }
 991
 992         /* if the object is no longer attached, discard its backing storage */
 993         if (obj->mm.madv == I915_MADV_DONTNEED &&
 994             !i915_gem_object_has_pages(obj))
 995                 i915_gem_object_truncate(obj);
 996
 997         args->retained = obj->mm.madv != __I915_MADV_PURGED;
 998         mutex_unlock(&obj->mm.lock);
 999
1000 out:
1001         i915_gem_object_put(obj);
1002         return err;
1003 }
1004
1005 int i915_gem_init(struct drm_i915_private *dev_priv)
1006 {
1007         int ret;
1008
1009         /* We need to fallback to 4K pages if host doesn't support huge gtt. */
1010         if (intel_vgpu_active(dev_priv) && !intel_vgpu_has_huge_gtt(dev_priv))
1011                 mkwrite_device_info(dev_priv)->page_sizes =
1012                         I915_GTT_PAGE_SIZE_4K;
1013
1014         ret = i915_gem_init_userptr(dev_priv);
1015         if (ret)
1016                 return ret;
1017
1018         intel_uc_fetch_firmwares(&dev_priv->gt.uc);
1019         intel_wopcm_init(&dev_priv->wopcm);
1020
1021         ret = i915_init_ggtt(dev_priv);
1022         if (ret) {
1023                 GEM_BUG_ON(ret == -EIO);
1024                 goto err_unlock;
1025         }
1026
1027         /*
1028          * Despite its name intel_init_clock_gating applies both display
1029          * clock gating workarounds; GT mmio workarounds and the occasional
1030          * GT power context workaround. Worse, sometimes it includes a context
1031          * register workaround which we need to apply before we record the
1032          * default HW state for all contexts.
1033          *
1034          * FIXME: break up the workarounds and apply them at the right time!
1035          */
1036         intel_init_clock_gating(dev_priv);
1037
1038         ret = intel_gt_init(&dev_priv->gt);
1039         if (ret)
1040                 goto err_unlock;
1041
1042         return 0;
1043
1044         /*
1045          * Unwinding is complicated by that we want to handle -EIO to mean
1046          * disable GPU submission but keep KMS alive. We want to mark the
1047          * HW as irrevisibly wedged, but keep enough state around that the
1048          * driver doesn't explode during runtime.
1049          */
1050 err_unlock:
1051         i915_gem_drain_workqueue(dev_priv);
1052
1053         if (ret != -EIO) {
1054                 intel_uc_cleanup_firmwares(&dev_priv->gt.uc);
1055                 i915_gem_cleanup_userptr(dev_priv);
1056         }
1057
1058         if (ret == -EIO) {
1059                 /*
1060                  * Allow engines or uC initialisation to fail by marking the GPU
1061                  * as wedged. But we only want to do this when the GPU is angry,
1062                  * for all other failure, such as an allocation failure, bail.
1063                  */
1064                 if (!intel_gt_is_wedged(&dev_priv->gt)) {
1065                         i915_probe_error(dev_priv,
1066                                          "Failed to initialize GPU, declaring it wedged!\n");
1067                         intel_gt_set_wedged(&dev_priv->gt);
1068                 }
1069
1070                 /* Minimal basic recovery for KMS */
1071                 ret = i915_ggtt_enable_hw(dev_priv);
1072                 i915_ggtt_resume(&dev_priv->ggtt);
1073                 intel_init_clock_gating(dev_priv);
1074         }
1075
1076         i915_gem_drain_freed_objects(dev_priv);
1077         return ret;
1078 }
1079
1080 void i915_gem_driver_register(struct drm_i915_private *i915)
1081 {
1082         i915_gem_driver_register__shrinker(i915);
1083
1084         intel_engines_driver_register(i915);
1085 }
1086
1087 void i915_gem_driver_unregister(struct drm_i915_private *i915)
1088 {
1089         i915_gem_driver_unregister__shrinker(i915);
1090 }
1091
1092 void i915_gem_driver_remove(struct drm_i915_private *dev_priv)
1093 {
1094         intel_wakeref_auto_fini(&dev_priv->ggtt.userfault_wakeref);
1095
1096         i915_gem_suspend_late(dev_priv);
1097         intel_gt_driver_remove(&dev_priv->gt);
1098         dev_priv->uabi_engines = RB_ROOT;
1099
1100         /* Flush any outstanding unpin_work. */
1101         i915_gem_drain_workqueue(dev_priv);
1102
1103         i915_gem_drain_freed_objects(dev_priv);
1104 }
1105
1106 void i915_gem_driver_release(struct drm_i915_private *dev_priv)
1107 {
1108         intel_gt_driver_release(&dev_priv->gt);
1109
1110         intel_wa_list_free(&dev_priv->gt_wa_list);
1111
1112         intel_uc_cleanup_firmwares(&dev_priv->gt.uc);
1113         i915_gem_cleanup_userptr(dev_priv);
1114
1115         i915_gem_drain_freed_objects(dev_priv);
1116
1117         drm_WARN_ON(&dev_priv->drm, !list_empty(&dev_priv->gem.contexts.list));
1118 }
1119
1120 static void i915_gem_init__mm(struct drm_i915_private *i915)
1121 {
1122         spin_lock_init(&i915->mm.obj_lock);
1123
1124         init_llist_head(&i915->mm.free_list);
1125
1126         INIT_LIST_HEAD(&i915->mm.purge_list);
1127         INIT_LIST_HEAD(&i915->mm.shrink_list);
1128
1129         i915_gem_init__objects(i915);
1130 }
1131
1132 void i915_gem_init_early(struct drm_i915_private *dev_priv)
1133 {
1134         i915_gem_init__mm(dev_priv);
1135         i915_gem_init__contexts(dev_priv);
1136
1137         spin_lock_init(&dev_priv->fb_tracking.lock);
1138 }
1139
1140 void i915_gem_cleanup_early(struct drm_i915_private *dev_priv)
1141 {
1142         i915_gem_drain_freed_objects(dev_priv);
1143         GEM_BUG_ON(!llist_empty(&dev_priv->mm.free_list));
1144         GEM_BUG_ON(atomic_read(&dev_priv->mm.free_count));
1145         drm_WARN_ON(&dev_priv->drm, dev_priv->mm.shrink_count);
1146 }
1147
1148 int i915_gem_open(struct drm_i915_private *i915, struct drm_file *file)
1149 {
1150         struct drm_i915_file_private *file_priv;
1151         int ret;
1152
1153         DRM_DEBUG("\n");
1154
1155         file_priv = kzalloc(sizeof(*file_priv), GFP_KERNEL);
1156         if (!file_priv)
1157                 return -ENOMEM;
1158
1159         file->driver_priv = file_priv;
1160         file_priv->dev_priv = i915;
1161         file_priv->file = file;
1162
1163         file_priv->bsd_engine = -1;
1164         file_priv->hang_timestamp = jiffies;
1165
1166         ret = i915_gem_context_open(i915, file);
1167         if (ret)
1168                 kfree(file_priv);
1169
1170         return ret;
1171 }
1172
1173 void i915_gem_ww_ctx_init(struct i915_gem_ww_ctx *ww, bool intr)
1174 {
1175         ww_acquire_init(&ww->ctx, &reservation_ww_class);
1176         INIT_LIST_HEAD(&ww->obj_list);
1177         ww->intr = intr;
1178         ww->contended = NULL;
1179 }
1180
1181 static void i915_gem_ww_ctx_unlock_all(struct i915_gem_ww_ctx *ww)
1182 {
1183         struct drm_i915_gem_object *obj;
1184
1185         while ((obj = list_first_entry_or_null(&ww->obj_list, struct drm_i915_gem_object, obj_link))) {
1186                 list_del(&obj->obj_link);
1187                 i915_gem_object_unlock(obj);
1188         }
1189 }
1190
1191 void i915_gem_ww_unlock_single(struct drm_i915_gem_object *obj)
1192 {
1193         list_del(&obj->obj_link);
1194         i915_gem_object_unlock(obj);
1195 }
1196
1197 void i915_gem_ww_ctx_fini(struct i915_gem_ww_ctx *ww)
1198 {
1199         i915_gem_ww_ctx_unlock_all(ww);
1200         WARN_ON(ww->contended);
1201         ww_acquire_fini(&ww->ctx);
1202 }
1203
1204 int __must_check i915_gem_ww_ctx_backoff(struct i915_gem_ww_ctx *ww)
1205 {
1206         int ret = 0;
1207
1208         if (WARN_ON(!ww->contended))
1209                 return -EINVAL;
1210
1211         i915_gem_ww_ctx_unlock_all(ww);
1212         if (ww->intr)
1213                 ret = dma_resv_lock_slow_interruptible(ww->contended->base.resv, &ww->ctx);
1214         else
1215                 dma_resv_lock_slow(ww->contended->base.resv, &ww->ctx);
1216
1217         if (!ret)
1218                 list_add_tail(&ww->contended->obj_link, &ww->obj_list);
1219
1220         ww->contended = NULL;
1221
1222         return ret;
1223 }
1224
1225 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
1226 #include "selftests/mock_gem_device.c"
1227 #include "selftests/i915_gem.c"
1228 #endif