drivers/gpu/drm/i915/i915_gem.c

   1 /*
   2  * Copyright © 2008-2015 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <[email protected]>
  25  *
  26  */
  27
  28 #include <drm/drmP.h>
  29 #include <drm/drm_vma_manager.h>
  30 #include <drm/i915_drm.h>
  31 #include "i915_drv.h"
  32 #include "i915_gem_dmabuf.h"
  33 #include "i915_vgpu.h"
  34 #include "i915_trace.h"
  35 #include "intel_drv.h"
  36 #include "intel_mocs.h"
  37 #include <linux/reservation.h>
  38 #include <linux/shmem_fs.h>
  39 #include <linux/slab.h>
  40 #include <linux/swap.h>
  41 #include <linux/pci.h>
  42 #include <linux/dma-buf.h>
  43
  44 static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
  45 static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
  46 static void
  47 i915_gem_object_retire__write(struct drm_i915_gem_object *obj);
  48 static void
  49 i915_gem_object_retire__read(struct drm_i915_gem_object *obj, int engine);
  50
  51 static bool cpu_cache_is_coherent(struct drm_device *dev,
  52                                   enum i915_cache_level level)
  53 {
  54         return HAS_LLC(dev) || level != I915_CACHE_NONE;
  55 }
  56
  57 static bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj)
  58 {
  59         if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
  60                 return false;
  61
  62         if (!cpu_cache_is_coherent(obj->base.dev, obj->cache_level))
  63                 return true;
  64
  65         return obj->pin_display;
  66 }
  67
  68 static int
  69 insert_mappable_node(struct drm_i915_private *i915,
  70                      struct drm_mm_node *node, u32 size)
  71 {
  72         memset(node, 0, sizeof(*node));
  73         return drm_mm_insert_node_in_range_generic(&i915->ggtt.base.mm, node,
  74                                                    size, 0, 0, 0,
  75                                                    i915->ggtt.mappable_end,
  76                                                    DRM_MM_SEARCH_DEFAULT,
  77                                                    DRM_MM_CREATE_DEFAULT);
  78 }
  79
  80 static void
  81 remove_mappable_node(struct drm_mm_node *node)
  82 {
  83         drm_mm_remove_node(node);
  84 }
  85
  86 /* some bookkeeping */
  87 static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
  88                                   size_t size)
  89 {
  90         spin_lock(&dev_priv->mm.object_stat_lock);
  91         dev_priv->mm.object_count++;
  92         dev_priv->mm.object_memory += size;
  93         spin_unlock(&dev_priv->mm.object_stat_lock);
  94 }
  95
  96 static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
  97                                      size_t size)
  98 {
  99         spin_lock(&dev_priv->mm.object_stat_lock);
 100         dev_priv->mm.object_count--;
 101         dev_priv->mm.object_memory -= size;
 102         spin_unlock(&dev_priv->mm.object_stat_lock);
 103 }
 104
 105 static int
 106 i915_gem_wait_for_error(struct i915_gpu_error *error)
 107 {
 108         int ret;
 109
 110         if (!i915_reset_in_progress(error))
 111                 return 0;
 112
 113         /*
 114          * Only wait 10 seconds for the gpu reset to complete to avoid hanging
 115          * userspace. If it takes that long something really bad is going on and
 116          * we should simply try to bail out and fail as gracefully as possible.
 117          */
 118         ret = wait_event_interruptible_timeout(error->reset_queue,
 119                                                !i915_reset_in_progress(error),
 120                                                10*HZ);
 121         if (ret == 0) {
 122                 DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
 123                 return -EIO;
 124         } else if (ret < 0) {
 125                 return ret;
 126         } else {
 127                 return 0;
 128         }
 129 }
 130
 131 int i915_mutex_lock_interruptible(struct drm_device *dev)
 132 {
 133         struct drm_i915_private *dev_priv = to_i915(dev);
 134         int ret;
 135
 136         ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
 137         if (ret)
 138                 return ret;
 139
 140         ret = mutex_lock_interruptible(&dev->struct_mutex);
 141         if (ret)
 142                 return ret;
 143
 144         WARN_ON(i915_verify_lists(dev));
 145         return 0;
 146 }
 147
 148 int
 149 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
 150                             struct drm_file *file)
 151 {
 152         struct drm_i915_private *dev_priv = to_i915(dev);
 153         struct i915_ggtt *ggtt = &dev_priv->ggtt;
 154         struct drm_i915_gem_get_aperture *args = data;
 155         struct i915_vma *vma;
 156         size_t pinned;
 157
 158         pinned = 0;
 159         mutex_lock(&dev->struct_mutex);
 160         list_for_each_entry(vma, &ggtt->base.active_list, vm_link)
 161                 if (vma->pin_count)
 162                         pinned += vma->node.size;
 163         list_for_each_entry(vma, &ggtt->base.inactive_list, vm_link)
 164                 if (vma->pin_count)
 165                         pinned += vma->node.size;
 166         mutex_unlock(&dev->struct_mutex);
 167
 168         args->aper_size = ggtt->base.total;
 169         args->aper_available_size = args->aper_size - pinned;
 170
 171         return 0;
 172 }
 173
 174 static int
 175 i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj)
 176 {
 177         struct address_space *mapping = file_inode(obj->base.filp)->i_mapping;
 178         char *vaddr = obj->phys_handle->vaddr;
 179         struct sg_table *st;
 180         struct scatterlist *sg;
 181         int i;
 182
 183         if (WARN_ON(i915_gem_object_needs_bit17_swizzle(obj)))
 184                 return -EINVAL;
 185
 186         for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
 187                 struct page *page;
 188                 char *src;
 189
 190                 page = shmem_read_mapping_page(mapping, i);
 191                 if (IS_ERR(page))
 192                         return PTR_ERR(page);
 193
 194                 src = kmap_atomic(page);
 195                 memcpy(vaddr, src, PAGE_SIZE);
 196                 drm_clflush_virt_range(vaddr, PAGE_SIZE);
 197                 kunmap_atomic(src);
 198
 199                 put_page(page);
 200                 vaddr += PAGE_SIZE;
 201         }
 202
 203         i915_gem_chipset_flush(to_i915(obj->base.dev));
 204
 205         st = kmalloc(sizeof(*st), GFP_KERNEL);
 206         if (st == NULL)
 207                 return -ENOMEM;
 208
 209         if (sg_alloc_table(st, 1, GFP_KERNEL)) {
 210                 kfree(st);
 211                 return -ENOMEM;
 212         }
 213
 214         sg = st->sgl;
 215         sg->offset = 0;
 216         sg->length = obj->base.size;
 217
 218         sg_dma_address(sg) = obj->phys_handle->busaddr;
 219         sg_dma_len(sg) = obj->base.size;
 220
 221         obj->pages = st;
 222         return 0;
 223 }
 224
 225 static void
 226 i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj)
 227 {
 228         int ret;
 229
 230         BUG_ON(obj->madv == __I915_MADV_PURGED);
 231
 232         ret = i915_gem_object_set_to_cpu_domain(obj, true);
 233         if (WARN_ON(ret)) {
 234                 /* In the event of a disaster, abandon all caches and
 235                  * hope for the best.
 236                  */
 237                 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
 238         }
 239
 240         if (obj->madv == I915_MADV_DONTNEED)
 241                 obj->dirty = 0;
 242
 243         if (obj->dirty) {
 244                 struct address_space *mapping = file_inode(obj->base.filp)->i_mapping;
 245                 char *vaddr = obj->phys_handle->vaddr;
 246                 int i;
 247
 248                 for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
 249                         struct page *page;
 250                         char *dst;
 251
 252                         page = shmem_read_mapping_page(mapping, i);
 253                         if (IS_ERR(page))
 254                                 continue;
 255
 256                         dst = kmap_atomic(page);
 257                         drm_clflush_virt_range(vaddr, PAGE_SIZE);
 258                         memcpy(dst, vaddr, PAGE_SIZE);
 259                         kunmap_atomic(dst);
 260
 261                         set_page_dirty(page);
 262                         if (obj->madv == I915_MADV_WILLNEED)
 263                                 mark_page_accessed(page);
 264                         put_page(page);
 265                         vaddr += PAGE_SIZE;
 266                 }
 267                 obj->dirty = 0;
 268         }
 269
 270         sg_free_table(obj->pages);
 271         kfree(obj->pages);
 272 }
 273
 274 static void
 275 i915_gem_object_release_phys(struct drm_i915_gem_object *obj)
 276 {
 277         drm_pci_free(obj->base.dev, obj->phys_handle);
 278 }
 279
 280 static const struct drm_i915_gem_object_ops i915_gem_phys_ops = {
 281         .get_pages = i915_gem_object_get_pages_phys,
 282         .put_pages = i915_gem_object_put_pages_phys,
 283         .release = i915_gem_object_release_phys,
 284 };
 285
 286 static int
 287 drop_pages(struct drm_i915_gem_object *obj)
 288 {
 289         struct i915_vma *vma, *next;
 290         int ret;
 291
 292         i915_gem_object_get(obj);
 293         list_for_each_entry_safe(vma, next, &obj->vma_list, obj_link)
 294                 if (i915_vma_unbind(vma))
 295                         break;
 296
 297         ret = i915_gem_object_put_pages(obj);
 298         i915_gem_object_put(obj);
 299
 300         return ret;
 301 }
 302
 303 int
 304 i915_gem_object_attach_phys(struct drm_i915_gem_object *obj,
 305                             int align)
 306 {
 307         drm_dma_handle_t *phys;
 308         int ret;
 309
 310         if (obj->phys_handle) {
 311                 if ((unsigned long)obj->phys_handle->vaddr & (align -1))
 312                         return -EBUSY;
 313
 314                 return 0;
 315         }
 316
 317         if (obj->madv != I915_MADV_WILLNEED)
 318                 return -EFAULT;
 319
 320         if (obj->base.filp == NULL)
 321                 return -EINVAL;
 322
 323         ret = drop_pages(obj);
 324         if (ret)
 325                 return ret;
 326
 327         /* create a new object */
 328         phys = drm_pci_alloc(obj->base.dev, obj->base.size, align);
 329         if (!phys)
 330                 return -ENOMEM;
 331
 332         obj->phys_handle = phys;
 333         obj->ops = &i915_gem_phys_ops;
 334
 335         return i915_gem_object_get_pages(obj);
 336 }
 337
 338 static int
 339 i915_gem_phys_pwrite(struct drm_i915_gem_object *obj,
 340                      struct drm_i915_gem_pwrite *args,
 341                      struct drm_file *file_priv)
 342 {
 343         struct drm_device *dev = obj->base.dev;
 344         void *vaddr = obj->phys_handle->vaddr + args->offset;
 345         char __user *user_data = u64_to_user_ptr(args->data_ptr);
 346         int ret = 0;
 347
 348         /* We manually control the domain here and pretend that it
 349          * remains coherent i.e. in the GTT domain, like shmem_pwrite.
 350          */
 351         ret = i915_gem_object_wait_rendering(obj, false);
 352         if (ret)
 353                 return ret;
 354
 355         intel_fb_obj_invalidate(obj, ORIGIN_CPU);
 356         if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) {
 357                 unsigned long unwritten;
 358
 359                 /* The physical object once assigned is fixed for the lifetime
 360                  * of the obj, so we can safely drop the lock and continue
 361                  * to access vaddr.
 362                  */
 363                 mutex_unlock(&dev->struct_mutex);
 364                 unwritten = copy_from_user(vaddr, user_data, args->size);
 365                 mutex_lock(&dev->struct_mutex);
 366                 if (unwritten) {
 367                         ret = -EFAULT;
 368                         goto out;
 369                 }
 370         }
 371
 372         drm_clflush_virt_range(vaddr, args->size);
 373         i915_gem_chipset_flush(to_i915(dev));
 374
 375 out:
 376         intel_fb_obj_flush(obj, false, ORIGIN_CPU);
 377         return ret;
 378 }
 379
 380 void *i915_gem_object_alloc(struct drm_device *dev)
 381 {
 382         struct drm_i915_private *dev_priv = to_i915(dev);
 383         return kmem_cache_zalloc(dev_priv->objects, GFP_KERNEL);
 384 }
 385
 386 void i915_gem_object_free(struct drm_i915_gem_object *obj)
 387 {
 388         struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
 389         kmem_cache_free(dev_priv->objects, obj);
 390 }
 391
 392 static int
 393 i915_gem_create(struct drm_file *file,
 394                 struct drm_device *dev,
 395                 uint64_t size,
 396                 uint32_t *handle_p)
 397 {
 398         struct drm_i915_gem_object *obj;
 399         int ret;
 400         u32 handle;
 401
 402         size = roundup(size, PAGE_SIZE);
 403         if (size == 0)
 404                 return -EINVAL;
 405
 406         /* Allocate the new object */
 407         obj = i915_gem_object_create(dev, size);
 408         if (IS_ERR(obj))
 409                 return PTR_ERR(obj);
 410
 411         ret = drm_gem_handle_create(file, &obj->base, &handle);
 412         /* drop reference from allocate - handle holds it now */
 413         i915_gem_object_put_unlocked(obj);
 414         if (ret)
 415                 return ret;
 416
 417         *handle_p = handle;
 418         return 0;
 419 }
 420
 421 int
 422 i915_gem_dumb_create(struct drm_file *file,
 423                      struct drm_device *dev,
 424                      struct drm_mode_create_dumb *args)
 425 {
 426         /* have to work out size/pitch and return them */
 427         args->pitch = ALIGN(args->width * DIV_ROUND_UP(args->bpp, 8), 64);
 428         args->size = args->pitch * args->height;
 429         return i915_gem_create(file, dev,
 430                                args->size, &args->handle);
 431 }
 432
 433 /**
 434  * Creates a new mm object and returns a handle to it.
 435  * @dev: drm device pointer
 436  * @data: ioctl data blob
 437  * @file: drm file pointer
 438  */
 439 int
 440 i915_gem_create_ioctl(struct drm_device *dev, void *data,
 441                       struct drm_file *file)
 442 {
 443         struct drm_i915_gem_create *args = data;
 444
 445         return i915_gem_create(file, dev,
 446                                args->size, &args->handle);
 447 }
 448
 449 static inline int
 450 __copy_to_user_swizzled(char __user *cpu_vaddr,
 451                         const char *gpu_vaddr, int gpu_offset,
 452                         int length)
 453 {
 454         int ret, cpu_offset = 0;
 455
 456         while (length > 0) {
 457                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
 458                 int this_length = min(cacheline_end - gpu_offset, length);
 459                 int swizzled_gpu_offset = gpu_offset ^ 64;
 460
 461                 ret = __copy_to_user(cpu_vaddr + cpu_offset,
 462                                      gpu_vaddr + swizzled_gpu_offset,
 463                                      this_length);
 464                 if (ret)
 465                         return ret + length;
 466
 467                 cpu_offset += this_length;
 468                 gpu_offset += this_length;
 469                 length -= this_length;
 470         }
 471
 472         return 0;
 473 }
 474
 475 static inline int
 476 __copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
 477                           const char __user *cpu_vaddr,
 478                           int length)
 479 {
 480         int ret, cpu_offset = 0;
 481
 482         while (length > 0) {
 483                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
 484                 int this_length = min(cacheline_end - gpu_offset, length);
 485                 int swizzled_gpu_offset = gpu_offset ^ 64;
 486
 487                 ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
 488                                        cpu_vaddr + cpu_offset,
 489                                        this_length);
 490                 if (ret)
 491                         return ret + length;
 492
 493                 cpu_offset += this_length;
 494                 gpu_offset += this_length;
 495                 length -= this_length;
 496         }
 497
 498         return 0;
 499 }
 500
 501 /*
 502  * Pins the specified object's pages and synchronizes the object with
 503  * GPU accesses. Sets needs_clflush to non-zero if the caller should
 504  * flush the object from the CPU cache.
 505  */
 506 int i915_gem_obj_prepare_shmem_read(struct drm_i915_gem_object *obj,
 507                                     int *needs_clflush)
 508 {
 509         int ret;
 510
 511         *needs_clflush = 0;
 512
 513         if (WARN_ON(!i915_gem_object_has_struct_page(obj)))
 514                 return -EINVAL;
 515
 516         ret = i915_gem_object_wait_rendering(obj, true);
 517         if (ret)
 518                 return ret;
 519
 520         if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) {
 521                 /* If we're not in the cpu read domain, set ourself into the gtt
 522                  * read domain and manually flush cachelines (if required). This
 523                  * optimizes for the case when the gpu will dirty the data
 524                  * anyway again before the next pread happens. */
 525                 *needs_clflush = !cpu_cache_is_coherent(obj->base.dev,
 526                                                         obj->cache_level);
 527         }
 528
 529         ret = i915_gem_object_get_pages(obj);
 530         if (ret)
 531                 return ret;
 532
 533         i915_gem_object_pin_pages(obj);
 534
 535         return ret;
 536 }
 537
 538 /* Per-page copy function for the shmem pread fastpath.
 539  * Flushes invalid cachelines before reading the target if
 540  * needs_clflush is set. */
 541 static int
 542 shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length,
 543                  char __user *user_data,
 544                  bool page_do_bit17_swizzling, bool needs_clflush)
 545 {
 546         char *vaddr;
 547         int ret;
 548
 549         if (unlikely(page_do_bit17_swizzling))
 550                 return -EINVAL;
 551
 552         vaddr = kmap_atomic(page);
 553         if (needs_clflush)
 554                 drm_clflush_virt_range(vaddr + shmem_page_offset,
 555                                        page_length);
 556         ret = __copy_to_user_inatomic(user_data,
 557                                       vaddr + shmem_page_offset,
 558                                       page_length);
 559         kunmap_atomic(vaddr);
 560
 561         return ret ? -EFAULT : 0;
 562 }
 563
 564 static void
 565 shmem_clflush_swizzled_range(char *addr, unsigned long length,
 566                              bool swizzled)
 567 {
 568         if (unlikely(swizzled)) {
 569                 unsigned long start = (unsigned long) addr;
 570                 unsigned long end = (unsigned long) addr + length;
 571
 572                 /* For swizzling simply ensure that we always flush both
 573                  * channels. Lame, but simple and it works. Swizzled
 574                  * pwrite/pread is far from a hotpath - current userspace
 575                  * doesn't use it at all. */
 576                 start = round_down(start, 128);
 577                 end = round_up(end, 128);
 578
 579                 drm_clflush_virt_range((void *)start, end - start);
 580         } else {
 581                 drm_clflush_virt_range(addr, length);
 582         }
 583
 584 }
 585
 586 /* Only difference to the fast-path function is that this can handle bit17
 587  * and uses non-atomic copy and kmap functions. */
 588 static int
 589 shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
 590                  char __user *user_data,
 591                  bool page_do_bit17_swizzling, bool needs_clflush)
 592 {
 593         char *vaddr;
 594         int ret;
 595
 596         vaddr = kmap(page);
 597         if (needs_clflush)
 598                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
 599                                              page_length,
 600                                              page_do_bit17_swizzling);
 601
 602         if (page_do_bit17_swizzling)
 603                 ret = __copy_to_user_swizzled(user_data,
 604                                               vaddr, shmem_page_offset,
 605                                               page_length);
 606         else
 607                 ret = __copy_to_user(user_data,
 608                                      vaddr + shmem_page_offset,
 609                                      page_length);
 610         kunmap(page);
 611
 612         return ret ? - EFAULT : 0;
 613 }
 614
 615 static inline unsigned long
 616 slow_user_access(struct io_mapping *mapping,
 617                  uint64_t page_base, int page_offset,
 618                  char __user *user_data,
 619                  unsigned long length, bool pwrite)
 620 {
 621         void __iomem *ioaddr;
 622         void *vaddr;
 623         uint64_t unwritten;
 624
 625         ioaddr = io_mapping_map_wc(mapping, page_base, PAGE_SIZE);
 626         /* We can use the cpu mem copy function because this is X86. */
 627         vaddr = (void __force *)ioaddr + page_offset;
 628         if (pwrite)
 629                 unwritten = __copy_from_user(vaddr, user_data, length);
 630         else
 631                 unwritten = __copy_to_user(user_data, vaddr, length);
 632
 633         io_mapping_unmap(ioaddr);
 634         return unwritten;
 635 }
 636
 637 static int
 638 i915_gem_gtt_pread(struct drm_device *dev,
 639                    struct drm_i915_gem_object *obj, uint64_t size,
 640                    uint64_t data_offset, uint64_t data_ptr)
 641 {
 642         struct drm_i915_private *dev_priv = to_i915(dev);
 643         struct i915_ggtt *ggtt = &dev_priv->ggtt;
 644         struct drm_mm_node node;
 645         char __user *user_data;
 646         uint64_t remain;
 647         uint64_t offset;
 648         int ret;
 649
 650         ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_MAPPABLE);
 651         if (ret) {
 652                 ret = insert_mappable_node(dev_priv, &node, PAGE_SIZE);
 653                 if (ret)
 654                         goto out;
 655
 656                 ret = i915_gem_object_get_pages(obj);
 657                 if (ret) {
 658                         remove_mappable_node(&node);
 659                         goto out;
 660                 }
 661
 662                 i915_gem_object_pin_pages(obj);
 663         } else {
 664                 node.start = i915_gem_obj_ggtt_offset(obj);
 665                 node.allocated = false;
 666                 ret = i915_gem_object_put_fence(obj);
 667                 if (ret)
 668                         goto out_unpin;
 669         }
 670
 671         ret = i915_gem_object_set_to_gtt_domain(obj, false);
 672         if (ret)
 673                 goto out_unpin;
 674
 675         user_data = u64_to_user_ptr(data_ptr);
 676         remain = size;
 677         offset = data_offset;
 678
 679         mutex_unlock(&dev->struct_mutex);
 680         if (likely(!i915.prefault_disable)) {
 681                 ret = fault_in_multipages_writeable(user_data, remain);
 682                 if (ret) {
 683                         mutex_lock(&dev->struct_mutex);
 684                         goto out_unpin;
 685                 }
 686         }
 687
 688         while (remain > 0) {
 689                 /* Operation in this page
 690                  *
 691                  * page_base = page offset within aperture
 692                  * page_offset = offset within page
 693                  * page_length = bytes to copy for this page
 694                  */
 695                 u32 page_base = node.start;
 696                 unsigned page_offset = offset_in_page(offset);
 697                 unsigned page_length = PAGE_SIZE - page_offset;
 698                 page_length = remain < page_length ? remain : page_length;
 699                 if (node.allocated) {
 700                         wmb();
 701                         ggtt->base.insert_page(&ggtt->base,
 702                                                i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT),
 703                                                node.start,
 704                                                I915_CACHE_NONE, 0);
 705                         wmb();
 706                 } else {
 707                         page_base += offset & PAGE_MASK;
 708                 }
 709                 /* This is a slow read/write as it tries to read from
 710                  * and write to user memory which may result into page
 711                  * faults, and so we cannot perform this under struct_mutex.
 712                  */
 713                 if (slow_user_access(ggtt->mappable, page_base,
 714                                      page_offset, user_data,
 715                                      page_length, false)) {
 716                         ret = -EFAULT;
 717                         break;
 718                 }
 719
 720                 remain -= page_length;
 721                 user_data += page_length;
 722                 offset += page_length;
 723         }
 724
 725         mutex_lock(&dev->struct_mutex);
 726         if (ret == 0 && (obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0) {
 727                 /* The user has modified the object whilst we tried
 728                  * reading from it, and we now have no idea what domain
 729                  * the pages should be in. As we have just been touching
 730                  * them directly, flush everything back to the GTT
 731                  * domain.
 732                  */
 733                 ret = i915_gem_object_set_to_gtt_domain(obj, false);
 734         }
 735
 736 out_unpin:
 737         if (node.allocated) {
 738                 wmb();
 739                 ggtt->base.clear_range(&ggtt->base,
 740                                        node.start, node.size,
 741                                        true);
 742                 i915_gem_object_unpin_pages(obj);
 743                 remove_mappable_node(&node);
 744         } else {
 745                 i915_gem_object_ggtt_unpin(obj);
 746         }
 747 out:
 748         return ret;
 749 }
 750
 751 static int
 752 i915_gem_shmem_pread(struct drm_device *dev,
 753                      struct drm_i915_gem_object *obj,
 754                      struct drm_i915_gem_pread *args,
 755                      struct drm_file *file)
 756 {
 757         char __user *user_data;
 758         ssize_t remain;
 759         loff_t offset;
 760         int shmem_page_offset, page_length, ret = 0;
 761         int obj_do_bit17_swizzling, page_do_bit17_swizzling;
 762         int prefaulted = 0;
 763         int needs_clflush = 0;
 764         struct sg_page_iter sg_iter;
 765
 766         if (!i915_gem_object_has_struct_page(obj))
 767                 return -ENODEV;
 768
 769         user_data = u64_to_user_ptr(args->data_ptr);
 770         remain = args->size;
 771
 772         obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 773
 774         ret = i915_gem_obj_prepare_shmem_read(obj, &needs_clflush);
 775         if (ret)
 776                 return ret;
 777
 778         offset = args->offset;
 779
 780         for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
 781                          offset >> PAGE_SHIFT) {
 782                 struct page *page = sg_page_iter_page(&sg_iter);
 783
 784                 if (remain <= 0)
 785                         break;
 786
 787                 /* Operation in this page
 788                  *
 789                  * shmem_page_offset = offset within page in shmem file
 790                  * page_length = bytes to copy for this page
 791                  */
 792                 shmem_page_offset = offset_in_page(offset);
 793                 page_length = remain;
 794                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
 795                         page_length = PAGE_SIZE - shmem_page_offset;
 796
 797                 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
 798                         (page_to_phys(page) & (1 << 17)) != 0;
 799
 800                 ret = shmem_pread_fast(page, shmem_page_offset, page_length,
 801                                        user_data, page_do_bit17_swizzling,
 802                                        needs_clflush);
 803                 if (ret == 0)
 804                         goto next_page;
 805
 806                 mutex_unlock(&dev->struct_mutex);
 807
 808                 if (likely(!i915.prefault_disable) && !prefaulted) {
 809                         ret = fault_in_multipages_writeable(user_data, remain);
 810                         /* Userspace is tricking us, but we've already clobbered
 811                          * its pages with the prefault and promised to write the
 812                          * data up to the first fault. Hence ignore any errors
 813                          * and just continue. */
 814                         (void)ret;
 815                         prefaulted = 1;
 816                 }
 817
 818                 ret = shmem_pread_slow(page, shmem_page_offset, page_length,
 819                                        user_data, page_do_bit17_swizzling,
 820                                        needs_clflush);
 821
 822                 mutex_lock(&dev->struct_mutex);
 823
 824                 if (ret)
 825                         goto out;
 826
 827 next_page:
 828                 remain -= page_length;
 829                 user_data += page_length;
 830                 offset += page_length;
 831         }
 832
 833 out:
 834         i915_gem_object_unpin_pages(obj);
 835
 836         return ret;
 837 }
 838
 839 /**
 840  * Reads data from the object referenced by handle.
 841  * @dev: drm device pointer
 842  * @data: ioctl data blob
 843  * @file: drm file pointer
 844  *
 845  * On error, the contents of *data are undefined.
 846  */
 847 int
 848 i915_gem_pread_ioctl(struct drm_device *dev, void *data,
 849                      struct drm_file *file)
 850 {
 851         struct drm_i915_gem_pread *args = data;
 852         struct drm_i915_gem_object *obj;
 853         int ret = 0;
 854
 855         if (args->size == 0)
 856                 return 0;
 857
 858         if (!access_ok(VERIFY_WRITE,
 859                        u64_to_user_ptr(args->data_ptr),
 860                        args->size))
 861                 return -EFAULT;
 862
 863         ret = i915_mutex_lock_interruptible(dev);
 864         if (ret)
 865                 return ret;
 866
 867         obj = i915_gem_object_lookup(file, args->handle);
 868         if (!obj) {
 869                 ret = -ENOENT;
 870                 goto unlock;
 871         }
 872
 873         /* Bounds check source.  */
 874         if (args->offset > obj->base.size ||
 875             args->size > obj->base.size - args->offset) {
 876                 ret = -EINVAL;
 877                 goto out;
 878         }
 879
 880         trace_i915_gem_object_pread(obj, args->offset, args->size);
 881
 882         ret = i915_gem_shmem_pread(dev, obj, args, file);
 883
 884         /* pread for non shmem backed objects */
 885         if (ret == -EFAULT || ret == -ENODEV)
 886                 ret = i915_gem_gtt_pread(dev, obj, args->size,
 887                                         args->offset, args->data_ptr);
 888
 889 out:
 890         i915_gem_object_put(obj);
 891 unlock:
 892         mutex_unlock(&dev->struct_mutex);
 893         return ret;
 894 }
 895
 896 /* This is the fast write path which cannot handle
 897  * page faults in the source data
 898  */
 899
 900 static inline int
 901 fast_user_write(struct io_mapping *mapping,
 902                 loff_t page_base, int page_offset,
 903                 char __user *user_data,
 904                 int length)
 905 {
 906         void __iomem *vaddr_atomic;
 907         void *vaddr;
 908         unsigned long unwritten;
 909
 910         vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
 911         /* We can use the cpu mem copy function because this is X86. */
 912         vaddr = (void __force*)vaddr_atomic + page_offset;
 913         unwritten = __copy_from_user_inatomic_nocache(vaddr,
 914                                                       user_data, length);
 915         io_mapping_unmap_atomic(vaddr_atomic);
 916         return unwritten;
 917 }
 918
 919 /**
 920  * This is the fast pwrite path, where we copy the data directly from the
 921  * user into the GTT, uncached.
 922  * @i915: i915 device private data
 923  * @obj: i915 gem object
 924  * @args: pwrite arguments structure
 925  * @file: drm file pointer
 926  */
 927 static int
 928 i915_gem_gtt_pwrite_fast(struct drm_i915_private *i915,
 929                          struct drm_i915_gem_object *obj,
 930                          struct drm_i915_gem_pwrite *args,
 931                          struct drm_file *file)
 932 {
 933         struct i915_ggtt *ggtt = &i915->ggtt;
 934         struct drm_device *dev = obj->base.dev;
 935         struct drm_mm_node node;
 936         uint64_t remain, offset;
 937         char __user *user_data;
 938         int ret;
 939         bool hit_slow_path = false;
 940
 941         if (obj->tiling_mode != I915_TILING_NONE)
 942                 return -EFAULT;
 943
 944         ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_MAPPABLE | PIN_NONBLOCK);
 945         if (ret) {
 946                 ret = insert_mappable_node(i915, &node, PAGE_SIZE);
 947                 if (ret)
 948                         goto out;
 949
 950                 ret = i915_gem_object_get_pages(obj);
 951                 if (ret) {
 952                         remove_mappable_node(&node);
 953                         goto out;
 954                 }
 955
 956                 i915_gem_object_pin_pages(obj);
 957         } else {
 958                 node.start = i915_gem_obj_ggtt_offset(obj);
 959                 node.allocated = false;
 960                 ret = i915_gem_object_put_fence(obj);
 961                 if (ret)
 962                         goto out_unpin;
 963         }
 964
 965         ret = i915_gem_object_set_to_gtt_domain(obj, true);
 966         if (ret)
 967                 goto out_unpin;
 968
 969         intel_fb_obj_invalidate(obj, ORIGIN_GTT);
 970         obj->dirty = true;
 971
 972         user_data = u64_to_user_ptr(args->data_ptr);
 973         offset = args->offset;
 974         remain = args->size;
 975         while (remain) {
 976                 /* Operation in this page
 977                  *
 978                  * page_base = page offset within aperture
 979                  * page_offset = offset within page
 980                  * page_length = bytes to copy for this page
 981                  */
 982                 u32 page_base = node.start;
 983                 unsigned page_offset = offset_in_page(offset);
 984                 unsigned page_length = PAGE_SIZE - page_offset;
 985                 page_length = remain < page_length ? remain : page_length;
 986                 if (node.allocated) {
 987                         wmb(); /* flush the write before we modify the GGTT */
 988                         ggtt->base.insert_page(&ggtt->base,
 989                                                i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT),
 990                                                node.start, I915_CACHE_NONE, 0);
 991                         wmb(); /* flush modifications to the GGTT (insert_page) */
 992                 } else {
 993                         page_base += offset & PAGE_MASK;
 994                 }
 995                 /* If we get a fault while copying data, then (presumably) our
 996                  * source page isn't available.  Return the error and we'll
 997                  * retry in the slow path.
 998                  * If the object is non-shmem backed, we retry again with the
 999                  * path that handles page fault.
1000                  */
1001                 if (fast_user_write(ggtt->mappable, page_base,
1002                                     page_offset, user_data, page_length)) {
1003                         hit_slow_path = true;
1004                         mutex_unlock(&dev->struct_mutex);
1005                         if (slow_user_access(ggtt->mappable,
1006                                              page_base,
1007                                              page_offset, user_data,
1008                                              page_length, true)) {
1009                                 ret = -EFAULT;
1010                                 mutex_lock(&dev->struct_mutex);
1011                                 goto out_flush;
1012                         }
1013
1014                         mutex_lock(&dev->struct_mutex);
1015                 }
1016
1017                 remain -= page_length;
1018                 user_data += page_length;
1019                 offset += page_length;
1020         }
1021
1022 out_flush:
1023         if (hit_slow_path) {
1024                 if (ret == 0 &&
1025                     (obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0) {
1026                         /* The user has modified the object whilst we tried
1027                          * reading from it, and we now have no idea what domain
1028                          * the pages should be in. As we have just been touching
1029                          * them directly, flush everything back to the GTT
1030                          * domain.
1031                          */
1032                         ret = i915_gem_object_set_to_gtt_domain(obj, false);
1033                 }
1034         }
1035
1036         intel_fb_obj_flush(obj, false, ORIGIN_GTT);
1037 out_unpin:
1038         if (node.allocated) {
1039                 wmb();
1040                 ggtt->base.clear_range(&ggtt->base,
1041                                        node.start, node.size,
1042                                        true);
1043                 i915_gem_object_unpin_pages(obj);
1044                 remove_mappable_node(&node);
1045         } else {
1046                 i915_gem_object_ggtt_unpin(obj);
1047         }
1048 out:
1049         return ret;
1050 }
1051
1052 /* Per-page copy function for the shmem pwrite fastpath.
1053  * Flushes invalid cachelines before writing to the target if
1054  * needs_clflush_before is set and flushes out any written cachelines after
1055  * writing if needs_clflush is set. */
1056 static int
1057 shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,
1058                   char __user *user_data,
1059                   bool page_do_bit17_swizzling,
1060                   bool needs_clflush_before,
1061                   bool needs_clflush_after)
1062 {
1063         char *vaddr;
1064         int ret;
1065
1066         if (unlikely(page_do_bit17_swizzling))
1067                 return -EINVAL;
1068
1069         vaddr = kmap_atomic(page);
1070         if (needs_clflush_before)
1071                 drm_clflush_virt_range(vaddr + shmem_page_offset,
1072                                        page_length);
1073         ret = __copy_from_user_inatomic(vaddr + shmem_page_offset,
1074                                         user_data, page_length);
1075         if (needs_clflush_after)
1076                 drm_clflush_virt_range(vaddr + shmem_page_offset,
1077                                        page_length);
1078         kunmap_atomic(vaddr);
1079
1080         return ret ? -EFAULT : 0;
1081 }
1082
1083 /* Only difference to the fast-path function is that this can handle bit17
1084  * and uses non-atomic copy and kmap functions. */
1085 static int
1086 shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
1087                   char __user *user_data,
1088                   bool page_do_bit17_swizzling,
1089                   bool needs_clflush_before,
1090                   bool needs_clflush_after)
1091 {
1092         char *vaddr;
1093         int ret;
1094
1095         vaddr = kmap(page);
1096         if (unlikely(needs_clflush_before || page_do_bit17_swizzling))
1097                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
1098                                              page_length,
1099                                              page_do_bit17_swizzling);
1100         if (page_do_bit17_swizzling)
1101                 ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
1102                                                 user_data,
1103                                                 page_length);
1104         else
1105                 ret = __copy_from_user(vaddr + shmem_page_offset,
1106                                        user_data,
1107                                        page_length);
1108         if (needs_clflush_after)
1109                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
1110                                              page_length,
1111                                              page_do_bit17_swizzling);
1112         kunmap(page);
1113
1114         return ret ? -EFAULT : 0;
1115 }
1116
1117 static int
1118 i915_gem_shmem_pwrite(struct drm_device *dev,
1119                       struct drm_i915_gem_object *obj,
1120                       struct drm_i915_gem_pwrite *args,
1121                       struct drm_file *file)
1122 {
1123         ssize_t remain;
1124         loff_t offset;
1125         char __user *user_data;
1126         int shmem_page_offset, page_length, ret = 0;
1127         int obj_do_bit17_swizzling, page_do_bit17_swizzling;
1128         int hit_slowpath = 0;
1129         int needs_clflush_after = 0;
1130         int needs_clflush_before = 0;
1131         struct sg_page_iter sg_iter;
1132
1133         user_data = u64_to_user_ptr(args->data_ptr);
1134         remain = args->size;
1135
1136         obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
1137
1138         ret = i915_gem_object_wait_rendering(obj, false);
1139         if (ret)
1140                 return ret;
1141
1142         if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
1143                 /* If we're not in the cpu write domain, set ourself into the gtt
1144                  * write domain and manually flush cachelines (if required). This
1145                  * optimizes for the case when the gpu will use the data
1146                  * right away and we therefore have to clflush anyway. */
1147                 needs_clflush_after = cpu_write_needs_clflush(obj);
1148         }
1149         /* Same trick applies to invalidate partially written cachelines read
1150          * before writing. */
1151         if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0)
1152                 needs_clflush_before =
1153                         !cpu_cache_is_coherent(dev, obj->cache_level);
1154
1155         ret = i915_gem_object_get_pages(obj);
1156         if (ret)
1157                 return ret;
1158
1159         intel_fb_obj_invalidate(obj, ORIGIN_CPU);
1160
1161         i915_gem_object_pin_pages(obj);
1162
1163         offset = args->offset;
1164         obj->dirty = 1;
1165
1166         for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
1167                          offset >> PAGE_SHIFT) {
1168                 struct page *page = sg_page_iter_page(&sg_iter);
1169                 int partial_cacheline_write;
1170
1171                 if (remain <= 0)
1172                         break;
1173
1174                 /* Operation in this page
1175                  *
1176                  * shmem_page_offset = offset within page in shmem file
1177                  * page_length = bytes to copy for this page
1178                  */
1179                 shmem_page_offset = offset_in_page(offset);
1180
1181                 page_length = remain;
1182                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
1183                         page_length = PAGE_SIZE - shmem_page_offset;
1184
1185                 /* If we don't overwrite a cacheline completely we need to be
1186                  * careful to have up-to-date data by first clflushing. Don't
1187                  * overcomplicate things and flush the entire patch. */
1188                 partial_cacheline_write = needs_clflush_before &&
1189                         ((shmem_page_offset | page_length)
1190                                 & (boot_cpu_data.x86_clflush_size - 1));
1191
1192                 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
1193                         (page_to_phys(page) & (1 << 17)) != 0;
1194
1195                 ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
1196                                         user_data, page_do_bit17_swizzling,
1197                                         partial_cacheline_write,
1198                                         needs_clflush_after);
1199                 if (ret == 0)
1200                         goto next_page;
1201
1202                 hit_slowpath = 1;
1203                 mutex_unlock(&dev->struct_mutex);
1204                 ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
1205                                         user_data, page_do_bit17_swizzling,
1206                                         partial_cacheline_write,
1207                                         needs_clflush_after);
1208
1209                 mutex_lock(&dev->struct_mutex);
1210
1211                 if (ret)
1212                         goto out;
1213
1214 next_page:
1215                 remain -= page_length;
1216                 user_data += page_length;
1217                 offset += page_length;
1218         }
1219
1220 out:
1221         i915_gem_object_unpin_pages(obj);
1222
1223         if (hit_slowpath) {
1224                 /*
1225                  * Fixup: Flush cpu caches in case we didn't flush the dirty
1226                  * cachelines in-line while writing and the object moved
1227                  * out of the cpu write domain while we've dropped the lock.
1228                  */
1229                 if (!needs_clflush_after &&
1230                     obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
1231                         if (i915_gem_clflush_object(obj, obj->pin_display))
1232                                 needs_clflush_after = true;
1233                 }
1234         }
1235
1236         if (needs_clflush_after)
1237                 i915_gem_chipset_flush(to_i915(dev));
1238         else
1239                 obj->cache_dirty = true;
1240
1241         intel_fb_obj_flush(obj, false, ORIGIN_CPU);
1242         return ret;
1243 }
1244
1245 /**
1246  * Writes data to the object referenced by handle.
1247  * @dev: drm device
1248  * @data: ioctl data blob
1249  * @file: drm file
1250  *
1251  * On error, the contents of the buffer that were to be modified are undefined.
1252  */
1253 int
1254 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
1255                       struct drm_file *file)
1256 {
1257         struct drm_i915_private *dev_priv = to_i915(dev);
1258         struct drm_i915_gem_pwrite *args = data;
1259         struct drm_i915_gem_object *obj;
1260         int ret;
1261
1262         if (args->size == 0)
1263                 return 0;
1264
1265         if (!access_ok(VERIFY_READ,
1266                        u64_to_user_ptr(args->data_ptr),
1267                        args->size))
1268                 return -EFAULT;
1269
1270         if (likely(!i915.prefault_disable)) {
1271                 ret = fault_in_multipages_readable(u64_to_user_ptr(args->data_ptr),
1272                                                    args->size);
1273                 if (ret)
1274                         return -EFAULT;
1275         }
1276
1277         intel_runtime_pm_get(dev_priv);
1278
1279         ret = i915_mutex_lock_interruptible(dev);
1280         if (ret)
1281                 goto put_rpm;
1282
1283         obj = i915_gem_object_lookup(file, args->handle);
1284         if (!obj) {
1285                 ret = -ENOENT;
1286                 goto unlock;
1287         }
1288
1289         /* Bounds check destination. */
1290         if (args->offset > obj->base.size ||
1291             args->size > obj->base.size - args->offset) {
1292                 ret = -EINVAL;
1293                 goto out;
1294         }
1295
1296         trace_i915_gem_object_pwrite(obj, args->offset, args->size);
1297
1298         ret = -EFAULT;
1299         /* We can only do the GTT pwrite on untiled buffers, as otherwise
1300          * it would end up going through the fenced access, and we'll get
1301          * different detiling behavior between reading and writing.
1302          * pread/pwrite currently are reading and writing from the CPU
1303          * perspective, requiring manual detiling by the client.
1304          */
1305         if (!i915_gem_object_has_struct_page(obj) ||
1306             cpu_write_needs_clflush(obj)) {
1307                 ret = i915_gem_gtt_pwrite_fast(dev_priv, obj, args, file);
1308                 /* Note that the gtt paths might fail with non-page-backed user
1309                  * pointers (e.g. gtt mappings when moving data between
1310                  * textures). Fallback to the shmem path in that case. */
1311         }
1312
1313         if (ret == -EFAULT || ret == -ENOSPC) {
1314                 if (obj->phys_handle)
1315                         ret = i915_gem_phys_pwrite(obj, args, file);
1316                 else if (i915_gem_object_has_struct_page(obj))
1317                         ret = i915_gem_shmem_pwrite(dev, obj, args, file);
1318                 else
1319                         ret = -ENODEV;
1320         }
1321
1322 out:
1323         i915_gem_object_put(obj);
1324 unlock:
1325         mutex_unlock(&dev->struct_mutex);
1326 put_rpm:
1327         intel_runtime_pm_put(dev_priv);
1328
1329         return ret;
1330 }
1331
1332 /**
1333  * Ensures that all rendering to the object has completed and the object is
1334  * safe to unbind from the GTT or access from the CPU.
1335  * @obj: i915 gem object
1336  * @readonly: waiting for read access or write
1337  */
1338 int
1339 i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
1340                                bool readonly)
1341 {
1342         struct reservation_object *resv;
1343         int ret, i;
1344
1345         if (readonly) {
1346                 if (obj->last_write_req != NULL) {
1347                         ret = i915_wait_request(obj->last_write_req);
1348                         if (ret)
1349                                 return ret;
1350
1351                         i = obj->last_write_req->engine->id;
1352                         if (obj->last_read_req[i] == obj->last_write_req)
1353                                 i915_gem_object_retire__read(obj, i);
1354                         else
1355                                 i915_gem_object_retire__write(obj);
1356                 }
1357         } else {
1358                 for (i = 0; i < I915_NUM_ENGINES; i++) {
1359                         if (obj->last_read_req[i] == NULL)
1360                                 continue;
1361
1362                         ret = i915_wait_request(obj->last_read_req[i]);
1363                         if (ret)
1364                                 return ret;
1365
1366                         i915_gem_object_retire__read(obj, i);
1367                 }
1368                 GEM_BUG_ON(obj->active);
1369         }
1370
1371         resv = i915_gem_object_get_dmabuf_resv(obj);
1372         if (resv) {
1373                 long err;
1374
1375                 err = reservation_object_wait_timeout_rcu(resv, !readonly, true,
1376                                                           MAX_SCHEDULE_TIMEOUT);
1377                 if (err < 0)
1378                         return err;
1379         }
1380
1381         return 0;
1382 }
1383
1384 static void
1385 i915_gem_object_retire_request(struct drm_i915_gem_object *obj,
1386                                struct drm_i915_gem_request *req)
1387 {
1388         int idx = req->engine->id;
1389
1390         if (obj->last_read_req[idx] == req)
1391                 i915_gem_object_retire__read(obj, idx);
1392         else if (obj->last_write_req == req)
1393                 i915_gem_object_retire__write(obj);
1394
1395         if (!i915_reset_in_progress(&req->i915->gpu_error))
1396                 i915_gem_request_retire_upto(req);
1397 }
1398
1399 /* A nonblocking variant of the above wait. This is a highly dangerous routine
1400  * as the object state may change during this call.
1401  */
1402 static __must_check int
1403 i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
1404                                             struct intel_rps_client *rps,
1405                                             bool readonly)
1406 {
1407         struct drm_device *dev = obj->base.dev;
1408         struct drm_i915_private *dev_priv = to_i915(dev);
1409         struct drm_i915_gem_request *requests[I915_NUM_ENGINES];
1410         int ret, i, n = 0;
1411
1412         BUG_ON(!mutex_is_locked(&dev->struct_mutex));
1413         BUG_ON(!dev_priv->mm.interruptible);
1414
1415         if (!obj->active)
1416                 return 0;
1417
1418         if (readonly) {
1419                 struct drm_i915_gem_request *req;
1420
1421                 req = obj->last_write_req;
1422                 if (req == NULL)
1423                         return 0;
1424
1425                 requests[n++] = i915_gem_request_get(req);
1426         } else {
1427                 for (i = 0; i < I915_NUM_ENGINES; i++) {
1428                         struct drm_i915_gem_request *req;
1429
1430                         req = obj->last_read_req[i];
1431                         if (req == NULL)
1432                                 continue;
1433
1434                         requests[n++] = i915_gem_request_get(req);
1435                 }
1436         }
1437
1438         mutex_unlock(&dev->struct_mutex);
1439         ret = 0;
1440         for (i = 0; ret == 0 && i < n; i++)
1441                 ret = __i915_wait_request(requests[i], true, NULL, rps);
1442         mutex_lock(&dev->struct_mutex);
1443
1444         for (i = 0; i < n; i++) {
1445                 if (ret == 0)
1446                         i915_gem_object_retire_request(obj, requests[i]);
1447                 i915_gem_request_put(requests[i]);
1448         }
1449
1450         return ret;
1451 }
1452
1453 static struct intel_rps_client *to_rps_client(struct drm_file *file)
1454 {
1455         struct drm_i915_file_private *fpriv = file->driver_priv;
1456         return &fpriv->rps;
1457 }
1458
1459 static enum fb_op_origin
1460 write_origin(struct drm_i915_gem_object *obj, unsigned domain)
1461 {
1462         return domain == I915_GEM_DOMAIN_GTT && !obj->has_wc_mmap ?
1463                ORIGIN_GTT : ORIGIN_CPU;
1464 }
1465
1466 /**
1467  * Called when user space prepares to use an object with the CPU, either
1468  * through the mmap ioctl's mapping or a GTT mapping.
1469  * @dev: drm device
1470  * @data: ioctl data blob
1471  * @file: drm file
1472  */
1473 int
1474 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1475                           struct drm_file *file)
1476 {
1477         struct drm_i915_gem_set_domain *args = data;
1478         struct drm_i915_gem_object *obj;
1479         uint32_t read_domains = args->read_domains;
1480         uint32_t write_domain = args->write_domain;
1481         int ret;
1482
1483         /* Only handle setting domains to types used by the CPU. */
1484         if (write_domain & I915_GEM_GPU_DOMAINS)
1485                 return -EINVAL;
1486
1487         if (read_domains & I915_GEM_GPU_DOMAINS)
1488                 return -EINVAL;
1489
1490         /* Having something in the write domain implies it's in the read
1491          * domain, and only that read domain.  Enforce that in the request.
1492          */
1493         if (write_domain != 0 && read_domains != write_domain)
1494                 return -EINVAL;
1495
1496         ret = i915_mutex_lock_interruptible(dev);
1497         if (ret)
1498                 return ret;
1499
1500         obj = i915_gem_object_lookup(file, args->handle);
1501         if (!obj) {
1502                 ret = -ENOENT;
1503                 goto unlock;
1504         }
1505
1506         /* Try to flush the object off the GPU without holding the lock.
1507          * We will repeat the flush holding the lock in the normal manner
1508          * to catch cases where we are gazumped.
1509          */
1510         ret = i915_gem_object_wait_rendering__nonblocking(obj,
1511                                                           to_rps_client(file),
1512                                                           !write_domain);
1513         if (ret)
1514                 goto unref;
1515
1516         if (read_domains & I915_GEM_DOMAIN_GTT)
1517                 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
1518         else
1519                 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
1520
1521         if (write_domain != 0)
1522                 intel_fb_obj_invalidate(obj, write_origin(obj, write_domain));
1523
1524 unref:
1525         i915_gem_object_put(obj);
1526 unlock:
1527         mutex_unlock(&dev->struct_mutex);
1528         return ret;
1529 }
1530
1531 /**
1532  * Called when user space has done writes to this buffer
1533  * @dev: drm device
1534  * @data: ioctl data blob
1535  * @file: drm file
1536  */
1537 int
1538 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1539                          struct drm_file *file)
1540 {
1541         struct drm_i915_gem_sw_finish *args = data;
1542         struct drm_i915_gem_object *obj;
1543         int ret = 0;
1544
1545         ret = i915_mutex_lock_interruptible(dev);
1546         if (ret)
1547                 return ret;
1548
1549         obj = i915_gem_object_lookup(file, args->handle);
1550         if (!obj) {
1551                 ret = -ENOENT;
1552                 goto unlock;
1553         }
1554
1555         /* Pinned buffers may be scanout, so flush the cache */
1556         if (obj->pin_display)
1557                 i915_gem_object_flush_cpu_write_domain(obj);
1558
1559         i915_gem_object_put(obj);
1560 unlock:
1561         mutex_unlock(&dev->struct_mutex);
1562         return ret;
1563 }
1564
1565 /**
1566  * i915_gem_mmap_ioctl - Maps the contents of an object, returning the address
1567  *                       it is mapped to.
1568  * @dev: drm device
1569  * @data: ioctl data blob
1570  * @file: drm file
1571  *
1572  * While the mapping holds a reference on the contents of the object, it doesn't
1573  * imply a ref on the object itself.
1574  *
1575  * IMPORTANT:
1576  *
1577  * DRM driver writers who look a this function as an example for how to do GEM
1578  * mmap support, please don't implement mmap support like here. The modern way
1579  * to implement DRM mmap support is with an mmap offset ioctl (like
1580  * i915_gem_mmap_gtt) and then using the mmap syscall on the DRM fd directly.
1581  * That way debug tooling like valgrind will understand what's going on, hiding
1582  * the mmap call in a driver private ioctl will break that. The i915 driver only
1583  * does cpu mmaps this way because we didn't know better.
1584  */
1585 int
1586 i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1587                     struct drm_file *file)
1588 {
1589         struct drm_i915_gem_mmap *args = data;
1590         struct drm_i915_gem_object *obj;
1591         unsigned long addr;
1592
1593         if (args->flags & ~(I915_MMAP_WC))
1594                 return -EINVAL;
1595
1596         if (args->flags & I915_MMAP_WC && !boot_cpu_has(X86_FEATURE_PAT))
1597                 return -ENODEV;
1598
1599         obj = i915_gem_object_lookup(file, args->handle);
1600         if (!obj)
1601                 return -ENOENT;
1602
1603         /* prime objects have no backing filp to GEM mmap
1604          * pages from.
1605          */
1606         if (!obj->base.filp) {
1607                 i915_gem_object_put_unlocked(obj);
1608                 return -EINVAL;
1609         }
1610
1611         addr = vm_mmap(obj->base.filp, 0, args->size,
1612                        PROT_READ | PROT_WRITE, MAP_SHARED,
1613                        args->offset);
1614         if (args->flags & I915_MMAP_WC) {
1615                 struct mm_struct *mm = current->mm;
1616                 struct vm_area_struct *vma;
1617
1618                 if (down_write_killable(&mm->mmap_sem)) {
1619                         i915_gem_object_put_unlocked(obj);
1620                         return -EINTR;
1621                 }
1622                 vma = find_vma(mm, addr);
1623                 if (vma)
1624                         vma->vm_page_prot =
1625                                 pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
1626                 else
1627                         addr = -ENOMEM;
1628                 up_write(&mm->mmap_sem);
1629
1630                 /* This may race, but that's ok, it only gets set */
1631                 WRITE_ONCE(obj->has_wc_mmap, true);
1632         }
1633         i915_gem_object_put_unlocked(obj);
1634         if (IS_ERR((void *)addr))
1635                 return addr;
1636
1637         args->addr_ptr = (uint64_t) addr;
1638
1639         return 0;
1640 }
1641
1642 /**
1643  * i915_gem_fault - fault a page into the GTT
1644  * @vma: VMA in question
1645  * @vmf: fault info
1646  *
1647  * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1648  * from userspace.  The fault handler takes care of binding the object to
1649  * the GTT (if needed), allocating and programming a fence register (again,
1650  * only if needed based on whether the old reg is still valid or the object
1651  * is tiled) and inserting a new PTE into the faulting process.
1652  *
1653  * Note that the faulting process may involve evicting existing objects
1654  * from the GTT and/or fence registers to make room.  So performance may
1655  * suffer if the GTT working set is large or there are few fence registers
1656  * left.
1657  */
1658 int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1659 {
1660         struct drm_i915_gem_object *obj = to_intel_bo(vma->vm_private_data);
1661         struct drm_device *dev = obj->base.dev;
1662         struct drm_i915_private *dev_priv = to_i915(dev);
1663         struct i915_ggtt *ggtt = &dev_priv->ggtt;
1664         struct i915_ggtt_view view = i915_ggtt_view_normal;
1665         pgoff_t page_offset;
1666         unsigned long pfn;
1667         int ret = 0;
1668         bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
1669
1670         intel_runtime_pm_get(dev_priv);
1671
1672         /* We don't use vmf->pgoff since that has the fake offset */
1673         page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
1674                 PAGE_SHIFT;
1675
1676         ret = i915_mutex_lock_interruptible(dev);
1677         if (ret)
1678                 goto out;
1679
1680         trace_i915_gem_object_fault(obj, page_offset, true, write);
1681
1682         /* Try to flush the object off the GPU first without holding the lock.
1683          * Upon reacquiring the lock, we will perform our sanity checks and then
1684          * repeat the flush holding the lock in the normal manner to catch cases
1685          * where we are gazumped.
1686          */
1687         ret = i915_gem_object_wait_rendering__nonblocking(obj, NULL, !write);
1688         if (ret)
1689                 goto unlock;
1690
1691         /* Access to snoopable pages through the GTT is incoherent. */
1692         if (obj->cache_level != I915_CACHE_NONE && !HAS_LLC(dev)) {
1693                 ret = -EFAULT;
1694                 goto unlock;
1695         }
1696
1697         /* Use a partial view if the object is bigger than the aperture. */
1698         if (obj->base.size >= ggtt->mappable_end &&
1699             obj->tiling_mode == I915_TILING_NONE) {
1700                 static const unsigned int chunk_size = 256; // 1 MiB
1701
1702                 memset(&view, 0, sizeof(view));
1703                 view.type = I915_GGTT_VIEW_PARTIAL;
1704                 view.params.partial.offset = rounddown(page_offset, chunk_size);
1705                 view.params.partial.size =
1706                         min_t(unsigned int,
1707                               chunk_size,
1708                               (vma->vm_end - vma->vm_start)/PAGE_SIZE -
1709                               view.params.partial.offset);
1710         }
1711
1712         /* Now pin it into the GTT if needed */
1713         ret = i915_gem_object_ggtt_pin(obj, &view, 0, PIN_MAPPABLE);
1714         if (ret)
1715                 goto unlock;
1716
1717         ret = i915_gem_object_set_to_gtt_domain(obj, write);
1718         if (ret)
1719                 goto unpin;
1720
1721         ret = i915_gem_object_get_fence(obj);
1722         if (ret)
1723                 goto unpin;
1724
1725         /* Finally, remap it using the new GTT offset */
1726         pfn = ggtt->mappable_base +
1727                 i915_gem_obj_ggtt_offset_view(obj, &view);
1728         pfn >>= PAGE_SHIFT;
1729
1730         if (unlikely(view.type == I915_GGTT_VIEW_PARTIAL)) {
1731                 /* Overriding existing pages in partial view does not cause
1732                  * us any trouble as TLBs are still valid because the fault
1733                  * is due to userspace losing part of the mapping or never
1734                  * having accessed it before (at this partials' range).
1735                  */
1736                 unsigned long base = vma->vm_start +
1737                                      (view.params.partial.offset << PAGE_SHIFT);
1738                 unsigned int i;
1739
1740                 for (i = 0; i < view.params.partial.size; i++) {
1741                         ret = vm_insert_pfn(vma, base + i * PAGE_SIZE, pfn + i);
1742                         if (ret)
1743                                 break;
1744                 }
1745
1746                 obj->fault_mappable = true;
1747         } else {
1748                 if (!obj->fault_mappable) {
1749                         unsigned long size = min_t(unsigned long,
1750                                                    vma->vm_end - vma->vm_start,
1751                                                    obj->base.size);
1752                         int i;
1753
1754                         for (i = 0; i < size >> PAGE_SHIFT; i++) {
1755                                 ret = vm_insert_pfn(vma,
1756                                                     (unsigned long)vma->vm_start + i * PAGE_SIZE,
1757                                                     pfn + i);
1758                                 if (ret)
1759                                         break;
1760                         }
1761
1762                         obj->fault_mappable = true;
1763                 } else
1764                         ret = vm_insert_pfn(vma,
1765                                             (unsigned long)vmf->virtual_address,
1766                                             pfn + page_offset);
1767         }
1768 unpin:
1769         i915_gem_object_ggtt_unpin_view(obj, &view);
1770 unlock:
1771         mutex_unlock(&dev->struct_mutex);
1772 out:
1773         switch (ret) {
1774         case -EIO:
1775                 /*
1776                  * We eat errors when the gpu is terminally wedged to avoid
1777                  * userspace unduly crashing (gl has no provisions for mmaps to
1778                  * fail). But any other -EIO isn't ours (e.g. swap in failure)
1779                  * and so needs to be reported.
1780                  */
1781                 if (!i915_terminally_wedged(&dev_priv->gpu_error)) {
1782                         ret = VM_FAULT_SIGBUS;
1783                         break;
1784                 }
1785         case -EAGAIN:
1786                 /*
1787                  * EAGAIN means the gpu is hung and we'll wait for the error
1788                  * handler to reset everything when re-faulting in
1789                  * i915_mutex_lock_interruptible.
1790                  */
1791         case 0:
1792         case -ERESTARTSYS:
1793         case -EINTR:
1794         case -EBUSY:
1795                 /*
1796                  * EBUSY is ok: this just means that another thread
1797                  * already did the job.
1798                  */
1799                 ret = VM_FAULT_NOPAGE;
1800                 break;
1801         case -ENOMEM:
1802                 ret = VM_FAULT_OOM;
1803                 break;
1804         case -ENOSPC:
1805         case -EFAULT:
1806                 ret = VM_FAULT_SIGBUS;
1807                 break;
1808         default:
1809                 WARN_ONCE(ret, "unhandled error in i915_gem_fault: %i\n", ret);
1810                 ret = VM_FAULT_SIGBUS;
1811                 break;
1812         }
1813
1814         intel_runtime_pm_put(dev_priv);
1815         return ret;
1816 }
1817
1818 /**
1819  * i915_gem_release_mmap - remove physical page mappings
1820  * @obj: obj in question
1821  *
1822  * Preserve the reservation of the mmapping with the DRM core code, but
1823  * relinquish ownership of the pages back to the system.
1824  *
1825  * It is vital that we remove the page mapping if we have mapped a tiled
1826  * object through the GTT and then lose the fence register due to
1827  * resource pressure. Similarly if the object has been moved out of the
1828  * aperture, than pages mapped into userspace must be revoked. Removing the
1829  * mapping will then trigger a page fault on the next user access, allowing
1830  * fixup by i915_gem_fault().
1831  */
1832 void
1833 i915_gem_release_mmap(struct drm_i915_gem_object *obj)
1834 {
1835         /* Serialisation between user GTT access and our code depends upon
1836          * revoking the CPU's PTE whilst the mutex is held. The next user
1837          * pagefault then has to wait until we release the mutex.
1838          */
1839         lockdep_assert_held(&obj->base.dev->struct_mutex);
1840
1841         if (!obj->fault_mappable)
1842                 return;
1843
1844         drm_vma_node_unmap(&obj->base.vma_node,
1845                            obj->base.dev->anon_inode->i_mapping);
1846
1847         /* Ensure that the CPU's PTE are revoked and there are not outstanding
1848          * memory transactions from userspace before we return. The TLB
1849          * flushing implied above by changing the PTE above *should* be
1850          * sufficient, an extra barrier here just provides us with a bit
1851          * of paranoid documentation about our requirement to serialise
1852          * memory writes before touching registers / GSM.
1853          */
1854         wmb();
1855
1856         obj->fault_mappable = false;
1857 }
1858
1859 void
1860 i915_gem_release_all_mmaps(struct drm_i915_private *dev_priv)
1861 {
1862         struct drm_i915_gem_object *obj;
1863
1864         list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list)
1865                 i915_gem_release_mmap(obj);
1866 }
1867
1868 uint32_t
1869 i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
1870 {
1871         uint32_t gtt_size;
1872
1873         if (INTEL_INFO(dev)->gen >= 4 ||
1874             tiling_mode == I915_TILING_NONE)
1875                 return size;
1876
1877         /* Previous chips need a power-of-two fence region when tiling */
1878         if (IS_GEN3(dev))
1879                 gtt_size = 1024*1024;
1880         else
1881                 gtt_size = 512*1024;
1882
1883         while (gtt_size < size)
1884                 gtt_size <<= 1;
1885
1886         return gtt_size;
1887 }
1888
1889 /**
1890  * i915_gem_get_gtt_alignment - return required GTT alignment for an object
1891  * @dev: drm device
1892  * @size: object size
1893  * @tiling_mode: tiling mode
1894  * @fenced: is fenced alignemned required or not
1895  *
1896  * Return the required GTT alignment for an object, taking into account
1897  * potential fence register mapping.
1898  */
1899 uint32_t
1900 i915_gem_get_gtt_alignment(struct drm_device *dev, uint32_t size,
1901                            int tiling_mode, bool fenced)
1902 {
1903         /*
1904          * Minimum alignment is 4k (GTT page size), but might be greater
1905          * if a fence register is needed for the object.
1906          */
1907         if (INTEL_INFO(dev)->gen >= 4 || (!fenced && IS_G33(dev)) ||
1908             tiling_mode == I915_TILING_NONE)
1909                 return 4096;
1910
1911         /*
1912          * Previous chips need to be aligned to the size of the smallest
1913          * fence register that can contain the object.
1914          */
1915         return i915_gem_get_gtt_size(dev, size, tiling_mode);
1916 }
1917
1918 static int i915_gem_object_create_mmap_offset(struct drm_i915_gem_object *obj)
1919 {
1920         struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
1921         int ret;
1922
1923         dev_priv->mm.shrinker_no_lock_stealing = true;
1924
1925         ret = drm_gem_create_mmap_offset(&obj->base);
1926         if (ret != -ENOSPC)
1927                 goto out;
1928
1929         /* Badly fragmented mmap space? The only way we can recover
1930          * space is by destroying unwanted objects. We can't randomly release
1931          * mmap_offsets as userspace expects them to be persistent for the
1932          * lifetime of the objects. The closest we can is to release the
1933          * offsets on purgeable objects by truncating it and marking it purged,
1934          * which prevents userspace from ever using that object again.
1935          */
1936         i915_gem_shrink(dev_priv,
1937                         obj->base.size >> PAGE_SHIFT,
1938                         I915_SHRINK_BOUND |
1939                         I915_SHRINK_UNBOUND |
1940                         I915_SHRINK_PURGEABLE);
1941         ret = drm_gem_create_mmap_offset(&obj->base);
1942         if (ret != -ENOSPC)
1943                 goto out;
1944
1945         i915_gem_shrink_all(dev_priv);
1946         ret = drm_gem_create_mmap_offset(&obj->base);
1947 out:
1948         dev_priv->mm.shrinker_no_lock_stealing = false;
1949
1950         return ret;
1951 }
1952
1953 static void i915_gem_object_free_mmap_offset(struct drm_i915_gem_object *obj)
1954 {
1955         drm_gem_free_mmap_offset(&obj->base);
1956 }
1957
1958 int
1959 i915_gem_mmap_gtt(struct drm_file *file,
1960                   struct drm_device *dev,
1961                   uint32_t handle,
1962                   uint64_t *offset)
1963 {
1964         struct drm_i915_gem_object *obj;
1965         int ret;
1966
1967         ret = i915_mutex_lock_interruptible(dev);
1968         if (ret)
1969                 return ret;
1970
1971         obj = i915_gem_object_lookup(file, handle);
1972         if (!obj) {
1973                 ret = -ENOENT;
1974                 goto unlock;
1975         }
1976
1977         if (obj->madv != I915_MADV_WILLNEED) {
1978                 DRM_DEBUG("Attempting to mmap a purgeable buffer\n");
1979                 ret = -EFAULT;
1980                 goto out;
1981         }
1982
1983         ret = i915_gem_object_create_mmap_offset(obj);
1984         if (ret)
1985                 goto out;
1986
1987         *offset = drm_vma_node_offset_addr(&obj->base.vma_node);
1988
1989 out:
1990         i915_gem_object_put(obj);
1991 unlock:
1992         mutex_unlock(&dev->struct_mutex);
1993         return ret;
1994 }
1995
1996 /**
1997  * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
1998  * @dev: DRM device
1999  * @data: GTT mapping ioctl data
2000  * @file: GEM object info
2001  *
2002  * Simply returns the fake offset to userspace so it can mmap it.
2003  * The mmap call will end up in drm_gem_mmap(), which will set things
2004  * up so we can get faults in the handler above.
2005  *
2006  * The fault handler will take care of binding the object into the GTT
2007  * (since it may have been evicted to make room for something), allocating
2008  * a fence register, and mapping the appropriate aperture address into
2009  * userspace.
2010  */
2011 int
2012 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
2013                         struct drm_file *file)
2014 {
2015         struct drm_i915_gem_mmap_gtt *args = data;
2016
2017         return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);
2018 }
2019
2020 /* Immediately discard the backing storage */
2021 static void
2022 i915_gem_object_truncate(struct drm_i915_gem_object *obj)
2023 {
2024         i915_gem_object_free_mmap_offset(obj);
2025
2026         if (obj->base.filp == NULL)
2027                 return;
2028
2029         /* Our goal here is to return as much of the memory as
2030          * is possible back to the system as we are called from OOM.
2031          * To do this we must instruct the shmfs to drop all of its
2032          * backing pages, *now*.
2033          */
2034         shmem_truncate_range(file_inode(obj->base.filp), 0, (loff_t)-1);
2035         obj->madv = __I915_MADV_PURGED;
2036 }
2037
2038 /* Try to discard unwanted pages */
2039 static void
2040 i915_gem_object_invalidate(struct drm_i915_gem_object *obj)
2041 {
2042         struct address_space *mapping;
2043
2044         switch (obj->madv) {
2045         case I915_MADV_DONTNEED:
2046                 i915_gem_object_truncate(obj);
2047         case __I915_MADV_PURGED:
2048                 return;
2049         }
2050
2051         if (obj->base.filp == NULL)
2052                 return;
2053
2054         mapping = file_inode(obj->base.filp)->i_mapping,
2055         invalidate_mapping_pages(mapping, 0, (loff_t)-1);
2056 }
2057
2058 static void
2059 i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
2060 {
2061         struct sgt_iter sgt_iter;
2062         struct page *page;
2063         int ret;
2064
2065         BUG_ON(obj->madv == __I915_MADV_PURGED);
2066
2067         ret = i915_gem_object_set_to_cpu_domain(obj, true);
2068         if (WARN_ON(ret)) {
2069                 /* In the event of a disaster, abandon all caches and
2070                  * hope for the best.
2071                  */
2072                 i915_gem_clflush_object(obj, true);
2073                 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2074         }
2075
2076         i915_gem_gtt_finish_object(obj);
2077
2078         if (i915_gem_object_needs_bit17_swizzle(obj))
2079                 i915_gem_object_save_bit_17_swizzle(obj);
2080
2081         if (obj->madv == I915_MADV_DONTNEED)
2082                 obj->dirty = 0;
2083
2084         for_each_sgt_page(page, sgt_iter, obj->pages) {
2085                 if (obj->dirty)
2086                         set_page_dirty(page);
2087
2088                 if (obj->madv == I915_MADV_WILLNEED)
2089                         mark_page_accessed(page);
2090
2091                 put_page(page);
2092         }
2093         obj->dirty = 0;
2094
2095         sg_free_table(obj->pages);
2096         kfree(obj->pages);
2097 }
2098
2099 int
2100 i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
2101 {
2102         const struct drm_i915_gem_object_ops *ops = obj->ops;
2103
2104         if (obj->pages == NULL)
2105                 return 0;
2106
2107         if (obj->pages_pin_count)
2108                 return -EBUSY;
2109
2110         BUG_ON(i915_gem_obj_bound_any(obj));
2111
2112         /* ->put_pages might need to allocate memory for the bit17 swizzle
2113          * array, hence protect them from being reaped by removing them from gtt
2114          * lists early. */
2115         list_del(&obj->global_list);
2116
2117         if (obj->mapping) {
2118                 if (is_vmalloc_addr(obj->mapping))
2119                         vunmap(obj->mapping);
2120                 else
2121                         kunmap(kmap_to_page(obj->mapping));
2122                 obj->mapping = NULL;
2123         }
2124
2125         ops->put_pages(obj);
2126         obj->pages = NULL;
2127
2128         i915_gem_object_invalidate(obj);
2129
2130         return 0;
2131 }
2132
2133 static int
2134 i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
2135 {
2136         struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
2137         int page_count, i;
2138         struct address_space *mapping;
2139         struct sg_table *st;
2140         struct scatterlist *sg;
2141         struct sgt_iter sgt_iter;
2142         struct page *page;
2143         unsigned long last_pfn = 0;     /* suppress gcc warning */
2144         int ret;
2145         gfp_t gfp;
2146
2147         /* Assert that the object is not currently in any GPU domain. As it
2148          * wasn't in the GTT, there shouldn't be any way it could have been in
2149          * a GPU cache
2150          */
2151         BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
2152         BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
2153
2154         st = kmalloc(sizeof(*st), GFP_KERNEL);
2155         if (st == NULL)
2156                 return -ENOMEM;
2157
2158         page_count = obj->base.size / PAGE_SIZE;
2159         if (sg_alloc_table(st, page_count, GFP_KERNEL)) {
2160                 kfree(st);
2161                 return -ENOMEM;
2162         }
2163
2164         /* Get the list of pages out of our struct file.  They'll be pinned
2165          * at this point until we release them.
2166          *
2167          * Fail silently without starting the shrinker
2168          */
2169         mapping = file_inode(obj->base.filp)->i_mapping;
2170         gfp = mapping_gfp_constraint(mapping, ~(__GFP_IO | __GFP_RECLAIM));
2171         gfp |= __GFP_NORETRY | __GFP_NOWARN;
2172         sg = st->sgl;
2173         st->nents = 0;
2174         for (i = 0; i < page_count; i++) {
2175                 page = shmem_read_mapping_page_gfp(mapping, i, gfp);
2176                 if (IS_ERR(page)) {
2177                         i915_gem_shrink(dev_priv,
2178                                         page_count,
2179                                         I915_SHRINK_BOUND |
2180                                         I915_SHRINK_UNBOUND |
2181                                         I915_SHRINK_PURGEABLE);
2182                         page = shmem_read_mapping_page_gfp(mapping, i, gfp);
2183                 }
2184                 if (IS_ERR(page)) {
2185                         /* We've tried hard to allocate the memory by reaping
2186                          * our own buffer, now let the real VM do its job and
2187                          * go down in flames if truly OOM.
2188                          */
2189                         i915_gem_shrink_all(dev_priv);
2190                         page = shmem_read_mapping_page(mapping, i);
2191                         if (IS_ERR(page)) {
2192                                 ret = PTR_ERR(page);
2193                                 goto err_pages;
2194                         }
2195                 }
2196 #ifdef CONFIG_SWIOTLB
2197                 if (swiotlb_nr_tbl()) {
2198                         st->nents++;
2199                         sg_set_page(sg, page, PAGE_SIZE, 0);
2200                         sg = sg_next(sg);
2201                         continue;
2202                 }
2203 #endif
2204                 if (!i || page_to_pfn(page) != last_pfn + 1) {
2205                         if (i)
2206                                 sg = sg_next(sg);
2207                         st->nents++;
2208                         sg_set_page(sg, page, PAGE_SIZE, 0);
2209                 } else {
2210                         sg->length += PAGE_SIZE;
2211                 }
2212                 last_pfn = page_to_pfn(page);
2213
2214                 /* Check that the i965g/gm workaround works. */
2215                 WARN_ON((gfp & __GFP_DMA32) && (last_pfn >= 0x00100000UL));
2216         }
2217 #ifdef CONFIG_SWIOTLB
2218         if (!swiotlb_nr_tbl())
2219 #endif
2220                 sg_mark_end(sg);
2221         obj->pages = st;
2222
2223         ret = i915_gem_gtt_prepare_object(obj);
2224         if (ret)
2225                 goto err_pages;
2226
2227         if (i915_gem_object_needs_bit17_swizzle(obj))
2228                 i915_gem_object_do_bit_17_swizzle(obj);
2229
2230         if (obj->tiling_mode != I915_TILING_NONE &&
2231             dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES)
2232                 i915_gem_object_pin_pages(obj);
2233
2234         return 0;
2235
2236 err_pages:
2237         sg_mark_end(sg);
2238         for_each_sgt_page(page, sgt_iter, st)
2239                 put_page(page);
2240         sg_free_table(st);
2241         kfree(st);
2242
2243         /* shmemfs first checks if there is enough memory to allocate the page
2244          * and reports ENOSPC should there be insufficient, along with the usual
2245          * ENOMEM for a genuine allocation failure.
2246          *
2247          * We use ENOSPC in our driver to mean that we have run out of aperture
2248          * space and so want to translate the error from shmemfs back to our
2249          * usual understanding of ENOMEM.
2250          */
2251         if (ret == -ENOSPC)
2252                 ret = -ENOMEM;
2253
2254         return ret;
2255 }
2256
2257 /* Ensure that the associated pages are gathered from the backing storage
2258  * and pinned into our object. i915_gem_object_get_pages() may be called
2259  * multiple times before they are released by a single call to
2260  * i915_gem_object_put_pages() - once the pages are no longer referenced
2261  * either as a result of memory pressure (reaping pages under the shrinker)
2262  * or as the object is itself released.
2263  */
2264 int
2265 i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
2266 {
2267         struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
2268         const struct drm_i915_gem_object_ops *ops = obj->ops;
2269         int ret;
2270
2271         if (obj->pages)
2272                 return 0;
2273
2274         if (obj->madv != I915_MADV_WILLNEED) {
2275                 DRM_DEBUG("Attempting to obtain a purgeable object\n");
2276                 return -EFAULT;
2277         }
2278
2279         BUG_ON(obj->pages_pin_count);
2280
2281         ret = ops->get_pages(obj);
2282         if (ret)
2283                 return ret;
2284
2285         list_add_tail(&obj->global_list, &dev_priv->mm.unbound_list);
2286
2287         obj->get_page.sg = obj->pages->sgl;
2288         obj->get_page.last = 0;
2289
2290         return 0;
2291 }
2292
2293 /* The 'mapping' part of i915_gem_object_pin_map() below */
2294 static void *i915_gem_object_map(const struct drm_i915_gem_object *obj)
2295 {
2296         unsigned long n_pages = obj->base.size >> PAGE_SHIFT;
2297         struct sg_table *sgt = obj->pages;
2298         struct sgt_iter sgt_iter;
2299         struct page *page;
2300         struct page *stack_pages[32];
2301         struct page **pages = stack_pages;
2302         unsigned long i = 0;
2303         void *addr;
2304
2305         /* A single page can always be kmapped */
2306         if (n_pages == 1)
2307                 return kmap(sg_page(sgt->sgl));
2308
2309         if (n_pages > ARRAY_SIZE(stack_pages)) {
2310                 /* Too big for stack -- allocate temporary array instead */
2311                 pages = drm_malloc_gfp(n_pages, sizeof(*pages), GFP_TEMPORARY);
2312                 if (!pages)
2313                         return NULL;
2314         }
2315
2316         for_each_sgt_page(page, sgt_iter, sgt)
2317                 pages[i++] = page;
2318
2319         /* Check that we have the expected number of pages */
2320         GEM_BUG_ON(i != n_pages);
2321
2322         addr = vmap(pages, n_pages, 0, PAGE_KERNEL);
2323
2324         if (pages != stack_pages)
2325                 drm_free_large(pages);
2326
2327         return addr;
2328 }
2329
2330 /* get, pin, and map the pages of the object into kernel space */
2331 void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj)
2332 {
2333         int ret;
2334
2335         lockdep_assert_held(&obj->base.dev->struct_mutex);
2336
2337         ret = i915_gem_object_get_pages(obj);
2338         if (ret)
2339                 return ERR_PTR(ret);
2340
2341         i915_gem_object_pin_pages(obj);
2342
2343         if (!obj->mapping) {
2344                 obj->mapping = i915_gem_object_map(obj);
2345                 if (!obj->mapping) {
2346                         i915_gem_object_unpin_pages(obj);
2347                         return ERR_PTR(-ENOMEM);
2348                 }
2349         }
2350
2351         return obj->mapping;
2352 }
2353
2354 void i915_vma_move_to_active(struct i915_vma *vma,
2355                              struct drm_i915_gem_request *req)
2356 {
2357         struct drm_i915_gem_object *obj = vma->obj;
2358         struct intel_engine_cs *engine;
2359
2360         engine = i915_gem_request_get_engine(req);
2361
2362         /* Add a reference if we're newly entering the active list. */
2363         if (obj->active == 0)
2364                 i915_gem_object_get(obj);
2365         obj->active |= intel_engine_flag(engine);
2366
2367         list_move_tail(&obj->engine_list[engine->id], &engine->active_list);
2368         i915_gem_request_assign(&obj->last_read_req[engine->id], req);
2369
2370         list_move_tail(&vma->vm_link, &vma->vm->active_list);
2371 }
2372
2373 static void
2374 i915_gem_object_retire__write(struct drm_i915_gem_object *obj)
2375 {
2376         GEM_BUG_ON(obj->last_write_req == NULL);
2377         GEM_BUG_ON(!(obj->active & intel_engine_flag(obj->last_write_req->engine)));
2378
2379         i915_gem_request_assign(&obj->last_write_req, NULL);
2380         intel_fb_obj_flush(obj, true, ORIGIN_CS);
2381 }
2382
2383 static void
2384 i915_gem_object_retire__read(struct drm_i915_gem_object *obj, int idx)
2385 {
2386         struct i915_vma *vma;
2387
2388         GEM_BUG_ON(obj->last_read_req[idx] == NULL);
2389         GEM_BUG_ON(!(obj->active & (1 << idx)));
2390
2391         list_del_init(&obj->engine_list[idx]);
2392         i915_gem_request_assign(&obj->last_read_req[idx], NULL);
2393
2394         if (obj->last_write_req && obj->last_write_req->engine->id == idx)
2395                 i915_gem_object_retire__write(obj);
2396
2397         obj->active &= ~(1 << idx);
2398         if (obj->active)
2399                 return;
2400
2401         /* Bump our place on the bound list to keep it roughly in LRU order
2402          * so that we don't steal from recently used but inactive objects
2403          * (unless we are forced to ofc!)
2404          */
2405         list_move_tail(&obj->global_list,
2406                        &to_i915(obj->base.dev)->mm.bound_list);
2407
2408         list_for_each_entry(vma, &obj->vma_list, obj_link) {
2409                 if (!list_empty(&vma->vm_link))
2410                         list_move_tail(&vma->vm_link, &vma->vm->inactive_list);
2411         }
2412
2413         i915_gem_request_assign(&obj->last_fenced_req, NULL);
2414         i915_gem_object_put(obj);
2415 }
2416
2417 static bool i915_context_is_banned(const struct i915_gem_context *ctx)
2418 {
2419         unsigned long elapsed;
2420
2421         if (ctx->hang_stats.banned)
2422                 return true;
2423
2424         elapsed = get_seconds() - ctx->hang_stats.guilty_ts;
2425         if (ctx->hang_stats.ban_period_seconds &&
2426             elapsed <= ctx->hang_stats.ban_period_seconds) {
2427                 DRM_DEBUG("context hanging too fast, banning!\n");
2428                 return true;
2429         }
2430
2431         return false;
2432 }
2433
2434 static void i915_set_reset_status(struct i915_gem_context *ctx,
2435                                   const bool guilty)
2436 {
2437         struct i915_ctx_hang_stats *hs = &ctx->hang_stats;
2438
2439         if (guilty) {
2440                 hs->banned = i915_context_is_banned(ctx);
2441                 hs->batch_active++;
2442                 hs->guilty_ts = get_seconds();
2443         } else {
2444                 hs->batch_pending++;
2445         }
2446 }
2447
2448 struct drm_i915_gem_request *
2449 i915_gem_find_active_request(struct intel_engine_cs *engine)
2450 {
2451         struct drm_i915_gem_request *request;
2452
2453         /* We are called by the error capture and reset at a random
2454          * point in time. In particular, note that neither is crucially
2455          * ordered with an interrupt. After a hang, the GPU is dead and we
2456          * assume that no more writes can happen (we waited long enough for
2457          * all writes that were in transaction to be flushed) - adding an
2458          * extra delay for a recent interrupt is pointless. Hence, we do
2459          * not need an engine->irq_seqno_barrier() before the seqno reads.
2460          */
2461         list_for_each_entry(request, &engine->request_list, list) {
2462                 if (i915_gem_request_completed(request))
2463                         continue;
2464
2465                 return request;
2466         }
2467
2468         return NULL;
2469 }
2470
2471 static void i915_gem_reset_engine_status(struct intel_engine_cs *engine)
2472 {
2473         struct drm_i915_gem_request *request;
2474         bool ring_hung;
2475
2476         request = i915_gem_find_active_request(engine);
2477         if (request == NULL)
2478                 return;
2479
2480         ring_hung = engine->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG;
2481
2482         i915_set_reset_status(request->ctx, ring_hung);
2483         list_for_each_entry_continue(request, &engine->request_list, list)
2484                 i915_set_reset_status(request->ctx, false);
2485 }
2486
2487 static void i915_gem_reset_engine_cleanup(struct intel_engine_cs *engine)
2488 {
2489         struct intel_ring *ring;
2490
2491         while (!list_empty(&engine->active_list)) {
2492                 struct drm_i915_gem_object *obj;
2493
2494                 obj = list_first_entry(&engine->active_list,
2495                                        struct drm_i915_gem_object,
2496                                        engine_list[engine->id]);
2497
2498                 i915_gem_object_retire__read(obj, engine->id);
2499         }
2500
2501         /* Mark all pending requests as complete so that any concurrent
2502          * (lockless) lookup doesn't try and wait upon the request as we
2503          * reset it.
2504          */
2505         intel_engine_init_seqno(engine, engine->last_submitted_seqno);
2506
2507         /*
2508          * Clear the execlists queue up before freeing the requests, as those
2509          * are the ones that keep the context and ringbuffer backing objects
2510          * pinned in place.
2511          */
2512
2513         if (i915.enable_execlists) {
2514                 /* Ensure irq handler finishes or is cancelled. */
2515                 tasklet_kill(&engine->irq_tasklet);
2516
2517                 intel_execlists_cancel_requests(engine);
2518         }
2519
2520         /*
2521          * We must free the requests after all the corresponding objects have
2522          * been moved off active lists. Which is the same order as the normal
2523          * retire_requests function does. This is important if object hold
2524          * implicit references on things like e.g. ppgtt address spaces through
2525          * the request.
2526          */
2527         if (!list_empty(&engine->request_list)) {
2528                 struct drm_i915_gem_request *request;
2529
2530                 request = list_last_entry(&engine->request_list,
2531                                           struct drm_i915_gem_request,
2532                                           list);
2533
2534                 i915_gem_request_retire_upto(request);
2535         }
2536
2537         /* Having flushed all requests from all queues, we know that all
2538          * ringbuffers must now be empty. However, since we do not reclaim
2539          * all space when retiring the request (to prevent HEADs colliding
2540          * with rapid ringbuffer wraparound) the amount of available space
2541          * upon reset is less than when we start. Do one more pass over
2542          * all the ringbuffers to reset last_retired_head.
2543          */
2544         list_for_each_entry(ring, &engine->buffers, link) {
2545                 ring->last_retired_head = ring->tail;
2546                 intel_ring_update_space(ring);
2547         }
2548
2549         engine->i915->gt.active_engines &= ~intel_engine_flag(engine);
2550 }
2551
2552 void i915_gem_reset(struct drm_device *dev)
2553 {
2554         struct drm_i915_private *dev_priv = to_i915(dev);
2555         struct intel_engine_cs *engine;
2556
2557         /*
2558          * Before we free the objects from the requests, we need to inspect
2559          * them for finding the guilty party. As the requests only borrow
2560          * their reference to the objects, the inspection must be done first.
2561          */
2562         for_each_engine(engine, dev_priv)
2563                 i915_gem_reset_engine_status(engine);
2564
2565         for_each_engine(engine, dev_priv)
2566                 i915_gem_reset_engine_cleanup(engine);
2567         mod_delayed_work(dev_priv->wq, &dev_priv->gt.idle_work, 0);
2568
2569         i915_gem_context_reset(dev);
2570
2571         i915_gem_restore_fences(dev);
2572
2573         WARN_ON(i915_verify_lists(dev));
2574 }
2575
2576 /**
2577  * This function clears the request list as sequence numbers are passed.
2578  * @engine: engine to retire requests on
2579  */
2580 void
2581 i915_gem_retire_requests_ring(struct intel_engine_cs *engine)
2582 {
2583         WARN_ON(i915_verify_lists(engine->dev));
2584
2585         /* Retire requests first as we use it above for the early return.
2586          * If we retire requests last, we may use a later seqno and so clear
2587          * the requests lists without clearing the active list, leading to
2588          * confusion.
2589          */
2590         while (!list_empty(&engine->request_list)) {
2591                 struct drm_i915_gem_request *request;
2592
2593                 request = list_first_entry(&engine->request_list,
2594                                            struct drm_i915_gem_request,
2595                                            list);
2596
2597                 if (!i915_gem_request_completed(request))
2598                         break;
2599
2600                 i915_gem_request_retire_upto(request);
2601         }
2602
2603         /* Move any buffers on the active list that are no longer referenced
2604          * by the ringbuffer to the flushing/inactive lists as appropriate,
2605          * before we free the context associated with the requests.
2606          */
2607         while (!list_empty(&engine->active_list)) {
2608                 struct drm_i915_gem_object *obj;
2609
2610                 obj = list_first_entry(&engine->active_list,
2611                                        struct drm_i915_gem_object,
2612                                        engine_list[engine->id]);
2613
2614                 if (!list_empty(&obj->last_read_req[engine->id]->list))
2615                         break;
2616
2617                 i915_gem_object_retire__read(obj, engine->id);
2618         }
2619
2620         WARN_ON(i915_verify_lists(engine->dev));
2621 }
2622
2623 void i915_gem_retire_requests(struct drm_i915_private *dev_priv)
2624 {
2625         struct intel_engine_cs *engine;
2626
2627         lockdep_assert_held(&dev_priv->drm.struct_mutex);
2628
2629         if (dev_priv->gt.active_engines == 0)
2630                 return;
2631
2632         GEM_BUG_ON(!dev_priv->gt.awake);
2633
2634         for_each_engine(engine, dev_priv) {
2635                 i915_gem_retire_requests_ring(engine);
2636                 if (list_empty(&engine->request_list))
2637                         dev_priv->gt.active_engines &= ~intel_engine_flag(engine);
2638         }
2639
2640         if (dev_priv->gt.active_engines == 0)
2641                 queue_delayed_work(dev_priv->wq,
2642                                    &dev_priv->gt.idle_work,
2643                                    msecs_to_jiffies(100));
2644 }
2645
2646 static void
2647 i915_gem_retire_work_handler(struct work_struct *work)
2648 {
2649         struct drm_i915_private *dev_priv =
2650                 container_of(work, typeof(*dev_priv), gt.retire_work.work);
2651         struct drm_device *dev = &dev_priv->drm;
2652
2653         /* Come back later if the device is busy... */
2654         if (mutex_trylock(&dev->struct_mutex)) {
2655                 i915_gem_retire_requests(dev_priv);
2656                 mutex_unlock(&dev->struct_mutex);
2657         }
2658
2659         /* Keep the retire handler running until we are finally idle.
2660          * We do not need to do this test under locking as in the worst-case
2661          * we queue the retire worker once too often.
2662          */
2663         if (READ_ONCE(dev_priv->gt.awake)) {
2664                 i915_queue_hangcheck(dev_priv);
2665                 queue_delayed_work(dev_priv->wq,
2666                                    &dev_priv->gt.retire_work,
2667                                    round_jiffies_up_relative(HZ));
2668         }
2669 }
2670
2671 static void
2672 i915_gem_idle_work_handler(struct work_struct *work)
2673 {
2674         struct drm_i915_private *dev_priv =
2675                 container_of(work, typeof(*dev_priv), gt.idle_work.work);
2676         struct drm_device *dev = &dev_priv->drm;
2677         struct intel_engine_cs *engine;
2678         unsigned int stuck_engines;
2679         bool rearm_hangcheck;
2680
2681         if (!READ_ONCE(dev_priv->gt.awake))
2682                 return;
2683
2684         if (READ_ONCE(dev_priv->gt.active_engines))
2685                 return;
2686
2687         rearm_hangcheck =
2688                 cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work);
2689
2690         if (!mutex_trylock(&dev->struct_mutex)) {
2691                 /* Currently busy, come back later */
2692                 mod_delayed_work(dev_priv->wq,
2693                                  &dev_priv->gt.idle_work,
2694                                  msecs_to_jiffies(50));
2695                 goto out_rearm;
2696         }
2697
2698         if (dev_priv->gt.active_engines)
2699                 goto out_unlock;
2700
2701         for_each_engine(engine, dev_priv)
2702                 i915_gem_batch_pool_fini(&engine->batch_pool);
2703
2704         GEM_BUG_ON(!dev_priv->gt.awake);
2705         dev_priv->gt.awake = false;
2706         rearm_hangcheck = false;
2707
2708         /* As we have disabled hangcheck, we need to unstick any waiters still
2709          * hanging around. However, as we may be racing against the interrupt
2710          * handler or the waiters themselves, we skip enabling the fake-irq.
2711          */
2712         stuck_engines = intel_kick_waiters(dev_priv);
2713         if (unlikely(stuck_engines))
2714                 DRM_DEBUG_DRIVER("kicked stuck waiters (%x)...missed irq?\n",
2715                                  stuck_engines);
2716
2717         if (INTEL_GEN(dev_priv) >= 6)
2718                 gen6_rps_idle(dev_priv);
2719         intel_runtime_pm_put(dev_priv);
2720 out_unlock:
2721         mutex_unlock(&dev->struct_mutex);
2722
2723 out_rearm:
2724         if (rearm_hangcheck) {
2725                 GEM_BUG_ON(!dev_priv->gt.awake);
2726                 i915_queue_hangcheck(dev_priv);
2727         }
2728 }
2729
2730 /**
2731  * Ensures that an object will eventually get non-busy by flushing any required
2732  * write domains, emitting any outstanding lazy request and retiring and
2733  * completed requests.
2734  * @obj: object to flush
2735  */
2736 static int
2737 i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
2738 {
2739         int i;
2740
2741         if (!obj->active)
2742                 return 0;
2743
2744         for (i = 0; i < I915_NUM_ENGINES; i++) {
2745                 struct drm_i915_gem_request *req;
2746
2747                 req = obj->last_read_req[i];
2748                 if (req == NULL)
2749                         continue;
2750
2751                 if (i915_gem_request_completed(req))
2752                         i915_gem_object_retire__read(obj, i);
2753         }
2754
2755         return 0;
2756 }
2757
2758 /**
2759  * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
2760  * @dev: drm device pointer
2761  * @data: ioctl data blob
2762  * @file: drm file pointer
2763  *
2764  * Returns 0 if successful, else an error is returned with the remaining time in
2765  * the timeout parameter.
2766  *  -ETIME: object is still busy after timeout
2767  *  -ERESTARTSYS: signal interrupted the wait
2768  *  -ENONENT: object doesn't exist
2769  * Also possible, but rare:
2770  *  -EAGAIN: GPU wedged
2771  *  -ENOMEM: damn
2772  *  -ENODEV: Internal IRQ fail
2773  *  -E?: The add request failed
2774  *
2775  * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
2776  * non-zero timeout parameter the wait ioctl will wait for the given number of
2777  * nanoseconds on an object becoming unbusy. Since the wait itself does so
2778  * without holding struct_mutex the object may become re-busied before this
2779  * function completes. A similar but shorter * race condition exists in the busy
2780  * ioctl
2781  */
2782 int
2783 i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
2784 {
2785         struct drm_i915_gem_wait *args = data;
2786         struct drm_i915_gem_object *obj;
2787         struct drm_i915_gem_request *req[I915_NUM_ENGINES];
2788         int i, n = 0;
2789         int ret;
2790
2791         if (args->flags != 0)
2792                 return -EINVAL;
2793
2794         ret = i915_mutex_lock_interruptible(dev);
2795         if (ret)
2796                 return ret;
2797
2798         obj = i915_gem_object_lookup(file, args->bo_handle);
2799         if (!obj) {
2800                 mutex_unlock(&dev->struct_mutex);
2801                 return -ENOENT;
2802         }
2803
2804         /* Need to make sure the object gets inactive eventually. */
2805         ret = i915_gem_object_flush_active(obj);
2806         if (ret)
2807                 goto out;
2808
2809         if (!obj->active)
2810                 goto out;
2811
2812         /* Do this after OLR check to make sure we make forward progress polling
2813          * on this IOCTL with a timeout == 0 (like busy ioctl)
2814          */
2815         if (args->timeout_ns == 0) {
2816                 ret = -ETIME;
2817                 goto out;
2818         }
2819
2820         i915_gem_object_put(obj);
2821
2822         for (i = 0; i < I915_NUM_ENGINES; i++) {
2823                 if (obj->last_read_req[i] == NULL)
2824                         continue;
2825
2826                 req[n++] = i915_gem_request_get(obj->last_read_req[i]);
2827         }
2828
2829         mutex_unlock(&dev->struct_mutex);
2830
2831         for (i = 0; i < n; i++) {
2832                 if (ret == 0)
2833                         ret = __i915_wait_request(req[i], true,
2834                                                   args->timeout_ns > 0 ? &args->timeout_ns : NULL,
2835                                                   to_rps_client(file));
2836                 i915_gem_request_put(req[i]);
2837         }
2838         return ret;
2839
2840 out:
2841         i915_gem_object_put(obj);
2842         mutex_unlock(&dev->struct_mutex);
2843         return ret;
2844 }
2845
2846 static int
2847 __i915_gem_object_sync(struct drm_i915_gem_object *obj,
2848                        struct drm_i915_gem_request *to,
2849                        struct drm_i915_gem_request *from)
2850 {
2851         int ret;
2852
2853         if (to->engine == from->engine)
2854                 return 0;
2855
2856         if (i915_gem_request_completed(from))
2857                 return 0;
2858
2859         if (!i915.semaphores) {
2860                 ret = __i915_wait_request(from,
2861                                           from->i915->mm.interruptible,
2862                                           NULL,
2863                                           NO_WAITBOOST);
2864                 if (ret)
2865                         return ret;
2866
2867                 i915_gem_object_retire_request(obj, from);
2868         } else {
2869                 int idx = intel_engine_sync_index(from->engine, to->engine);
2870                 if (from->fence.seqno <= from->engine->semaphore.sync_seqno[idx])
2871                         return 0;
2872
2873                 trace_i915_gem_ring_sync_to(to, from);
2874                 ret = to->engine->semaphore.sync_to(to, from);
2875                 if (ret)
2876                         return ret;
2877
2878                 from->engine->semaphore.sync_seqno[idx] = from->fence.seqno;
2879         }
2880
2881         return 0;
2882 }
2883
2884 /**
2885  * i915_gem_object_sync - sync an object to a ring.
2886  *
2887  * @obj: object which may be in use on another ring.
2888  * @to: request we are wishing to use
2889  *
2890  * This code is meant to abstract object synchronization with the GPU.
2891  * Conceptually we serialise writes between engines inside the GPU.
2892  * We only allow one engine to write into a buffer at any time, but
2893  * multiple readers. To ensure each has a coherent view of memory, we must:
2894  *
2895  * - If there is an outstanding write request to the object, the new
2896  *   request must wait for it to complete (either CPU or in hw, requests
2897  *   on the same ring will be naturally ordered).
2898  *
2899  * - If we are a write request (pending_write_domain is set), the new
2900  *   request must wait for outstanding read requests to complete.
2901  *
2902  * Returns 0 if successful, else propagates up the lower layer error.
2903  */
2904 int
2905 i915_gem_object_sync(struct drm_i915_gem_object *obj,
2906                      struct drm_i915_gem_request *to)
2907 {
2908         const bool readonly = obj->base.pending_write_domain == 0;
2909         struct drm_i915_gem_request *req[I915_NUM_ENGINES];
2910         int ret, i, n;
2911
2912         if (!obj->active)
2913                 return 0;
2914
2915         n = 0;
2916         if (readonly) {
2917                 if (obj->last_write_req)
2918                         req[n++] = obj->last_write_req;
2919         } else {
2920                 for (i = 0; i < I915_NUM_ENGINES; i++)
2921                         if (obj->last_read_req[i])
2922                                 req[n++] = obj->last_read_req[i];
2923         }
2924         for (i = 0; i < n; i++) {
2925                 ret = __i915_gem_object_sync(obj, to, req[i]);
2926                 if (ret)
2927                         return ret;
2928         }
2929
2930         return 0;
2931 }
2932
2933 static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
2934 {
2935         u32 old_write_domain, old_read_domains;
2936
2937         /* Force a pagefault for domain tracking on next user access */
2938         i915_gem_release_mmap(obj);
2939
2940         if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
2941                 return;
2942
2943         old_read_domains = obj->base.read_domains;
2944         old_write_domain = obj->base.write_domain;
2945
2946         obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
2947         obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
2948
2949         trace_i915_gem_object_change_domain(obj,
2950                                             old_read_domains,
2951                                             old_write_domain);
2952 }
2953
2954 static void __i915_vma_iounmap(struct i915_vma *vma)
2955 {
2956         GEM_BUG_ON(vma->pin_count);
2957
2958         if (vma->iomap == NULL)
2959                 return;
2960
2961         io_mapping_unmap(vma->iomap);
2962         vma->iomap = NULL;
2963 }
2964
2965 static int __i915_vma_unbind(struct i915_vma *vma, bool wait)
2966 {
2967         struct drm_i915_gem_object *obj = vma->obj;
2968         struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
2969         int ret;
2970
2971         if (list_empty(&vma->obj_link))
2972                 return 0;
2973
2974         if (!drm_mm_node_allocated(&vma->node)) {
2975                 i915_gem_vma_destroy(vma);
2976                 return 0;
2977         }
2978
2979         if (vma->pin_count)
2980                 return -EBUSY;
2981
2982         BUG_ON(obj->pages == NULL);
2983
2984         if (wait) {
2985                 ret = i915_gem_object_wait_rendering(obj, false);
2986                 if (ret)
2987                         return ret;
2988         }
2989
2990         if (vma->is_ggtt && vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL) {
2991                 i915_gem_object_finish_gtt(obj);
2992
2993                 /* release the fence reg _after_ flushing */
2994                 ret = i915_gem_object_put_fence(obj);
2995                 if (ret)
2996                         return ret;
2997
2998                 __i915_vma_iounmap(vma);
2999         }
3000
3001         trace_i915_vma_unbind(vma);
3002
3003         vma->vm->unbind_vma(vma);
3004         vma->bound = 0;
3005
3006         list_del_init(&vma->vm_link);
3007         if (vma->is_ggtt) {
3008                 if (vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL) {
3009                         obj->map_and_fenceable = false;
3010                 } else if (vma->ggtt_view.pages) {
3011                         sg_free_table(vma->ggtt_view.pages);
3012                         kfree(vma->ggtt_view.pages);
3013                 }
3014                 vma->ggtt_view.pages = NULL;
3015         }
3016
3017         drm_mm_remove_node(&vma->node);
3018         i915_gem_vma_destroy(vma);
3019
3020         /* Since the unbound list is global, only move to that list if
3021          * no more VMAs exist. */
3022         if (list_empty(&obj->vma_list))
3023                 list_move_tail(&obj->global_list, &dev_priv->mm.unbound_list);
3024
3025         /* And finally now the object is completely decoupled from this vma,
3026          * we can drop its hold on the backing storage and allow it to be
3027          * reaped by the shrinker.
3028          */
3029         i915_gem_object_unpin_pages(obj);
3030
3031         return 0;
3032 }
3033
3034 int i915_vma_unbind(struct i915_vma *vma)
3035 {
3036         return __i915_vma_unbind(vma, true);
3037 }
3038
3039 int __i915_vma_unbind_no_wait(struct i915_vma *vma)
3040 {
3041         return __i915_vma_unbind(vma, false);
3042 }
3043
3044 int i915_gem_wait_for_idle(struct drm_i915_private *dev_priv)
3045 {
3046         struct intel_engine_cs *engine;
3047         int ret;
3048
3049         lockdep_assert_held(&dev_priv->drm.struct_mutex);
3050
3051         for_each_engine(engine, dev_priv) {
3052                 if (engine->last_context == NULL)
3053                         continue;
3054
3055                 ret = intel_engine_idle(engine);
3056                 if (ret)
3057                         return ret;
3058         }
3059
3060         WARN_ON(i915_verify_lists(dev));
3061         return 0;
3062 }
3063
3064 static bool i915_gem_valid_gtt_space(struct i915_vma *vma,
3065                                      unsigned long cache_level)
3066 {
3067         struct drm_mm_node *gtt_space = &vma->node;
3068         struct drm_mm_node *other;
3069
3070         /*
3071          * On some machines we have to be careful when putting differing types
3072          * of snoopable memory together to avoid the prefetcher crossing memory
3073          * domains and dying. During vm initialisation, we decide whether or not
3074          * these constraints apply and set the drm_mm.color_adjust
3075          * appropriately.
3076          */
3077         if (vma->vm->mm.color_adjust == NULL)
3078                 return true;
3079
3080         if (!drm_mm_node_allocated(gtt_space))
3081                 return true;
3082
3083         if (list_empty(&gtt_space->node_list))
3084                 return true;
3085
3086         other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list);
3087         if (other->allocated && !other->hole_follows && other->color != cache_level)
3088                 return false;
3089
3090         other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list);
3091         if (other->allocated && !gtt_space->hole_follows && other->color != cache_level)
3092                 return false;
3093
3094         return true;
3095 }
3096
3097 /**
3098  * Finds free space in the GTT aperture and binds the object or a view of it
3099  * there.
3100  * @obj: object to bind
3101  * @vm: address space to bind into
3102  * @ggtt_view: global gtt view if applicable
3103  * @alignment: requested alignment
3104  * @flags: mask of PIN_* flags to use
3105  */
3106 static struct i915_vma *
3107 i915_gem_object_bind_to_vm(struct drm_i915_gem_object *obj,
3108                            struct i915_address_space *vm,
3109                            const struct i915_ggtt_view *ggtt_view,
3110                            unsigned alignment,
3111                            uint64_t flags)
3112 {
3113         struct drm_device *dev = obj->base.dev;
3114         struct drm_i915_private *dev_priv = to_i915(dev);
3115         struct i915_ggtt *ggtt = &dev_priv->ggtt;
3116         u32 fence_alignment, unfenced_alignment;
3117         u32 search_flag, alloc_flag;
3118         u64 start, end;
3119         u64 size, fence_size;
3120         struct i915_vma *vma;
3121         int ret;
3122
3123         if (i915_is_ggtt(vm)) {
3124                 u32 view_size;
3125
3126                 if (WARN_ON(!ggtt_view))
3127                         return ERR_PTR(-EINVAL);
3128
3129                 view_size = i915_ggtt_view_size(obj, ggtt_view);
3130
3131                 fence_size = i915_gem_get_gtt_size(dev,
3132                                                    view_size,
3133                                                    obj->tiling_mode);
3134                 fence_alignment = i915_gem_get_gtt_alignment(dev,
3135                                                              view_size,
3136                                                              obj->tiling_mode,
3137                                                              true);
3138                 unfenced_alignment = i915_gem_get_gtt_alignment(dev,
3139                                                                 view_size,
3140                                                                 obj->tiling_mode,
3141                                                                 false);
3142                 size = flags & PIN_MAPPABLE ? fence_size : view_size;
3143         } else {
3144                 fence_size = i915_gem_get_gtt_size(dev,
3145                                                    obj->base.size,
3146                                                    obj->tiling_mode);
3147                 fence_alignment = i915_gem_get_gtt_alignment(dev,
3148                                                              obj->base.size,
3149                                                              obj->tiling_mode,
3150                                                              true);
3151                 unfenced_alignment =
3152                         i915_gem_get_gtt_alignment(dev,
3153                                                    obj->base.size,
3154                                                    obj->tiling_mode,
3155                                                    false);
3156                 size = flags & PIN_MAPPABLE ? fence_size : obj->base.size;
3157         }
3158
3159         start = flags & PIN_OFFSET_BIAS ? flags & PIN_OFFSET_MASK : 0;
3160         end = vm->total;
3161         if (flags & PIN_MAPPABLE)
3162                 end = min_t(u64, end, ggtt->mappable_end);
3163         if (flags & PIN_ZONE_4G)
3164                 end = min_t(u64, end, (1ULL << 32) - PAGE_SIZE);
3165
3166         if (alignment == 0)
3167                 alignment = flags & PIN_MAPPABLE ? fence_alignment :
3168                                                 unfenced_alignment;
3169         if (flags & PIN_MAPPABLE && alignment & (fence_alignment - 1)) {
3170                 DRM_DEBUG("Invalid object (view type=%u) alignment requested %u\n",
3171                           ggtt_view ? ggtt_view->type : 0,
3172                           alignment);
3173                 return ERR_PTR(-EINVAL);
3174         }
3175
3176         /* If binding the object/GGTT view requires more space than the entire
3177          * aperture has, reject it early before evicting everything in a vain
3178          * attempt to find space.
3179          */
3180         if (size > end) {
3181                 DRM_DEBUG("Attempting to bind an object (view type=%u) larger than the aperture: size=%llu > %s aperture=%llu\n",
3182                           ggtt_view ? ggtt_view->type : 0,
3183                           size,
3184                           flags & PIN_MAPPABLE ? "mappable" : "total",
3185                           end);
3186                 return ERR_PTR(-E2BIG);
3187         }
3188
3189         ret = i915_gem_object_get_pages(obj);
3190         if (ret)
3191                 return ERR_PTR(ret);
3192
3193         i915_gem_object_pin_pages(obj);
3194
3195         vma = ggtt_view ? i915_gem_obj_lookup_or_create_ggtt_vma(obj, ggtt_view) :
3196                           i915_gem_obj_lookup_or_create_vma(obj, vm);
3197
3198         if (IS_ERR(vma))
3199                 goto err_unpin;
3200
3201         if (flags & PIN_OFFSET_FIXED) {
3202                 uint64_t offset = flags & PIN_OFFSET_MASK;
3203
3204                 if (offset & (alignment - 1) || offset + size > end) {
3205                         ret = -EINVAL;
3206                         goto err_free_vma;
3207                 }
3208                 vma->node.start = offset;
3209                 vma->node.size = size;
3210                 vma->node.color = obj->cache_level;
3211                 ret = drm_mm_reserve_node(&vm->mm, &vma->node);
3212                 if (ret) {
3213                         ret = i915_gem_evict_for_vma(vma);
3214                         if (ret == 0)
3215                                 ret = drm_mm_reserve_node(&vm->mm, &vma->node);
3216                 }
3217                 if (ret)
3218                         goto err_free_vma;
3219         } else {
3220                 if (flags & PIN_HIGH) {
3221                         search_flag = DRM_MM_SEARCH_BELOW;
3222                         alloc_flag = DRM_MM_CREATE_TOP;
3223                 } else {
3224                         search_flag = DRM_MM_SEARCH_DEFAULT;
3225                         alloc_flag = DRM_MM_CREATE_DEFAULT;
3226                 }
3227
3228 search_free:
3229                 ret = drm_mm_insert_node_in_range_generic(&vm->mm, &vma->node,
3230                                                           size, alignment,
3231                                                           obj->cache_level,
3232                                                           start, end,
3233                                                           search_flag,
3234                                                           alloc_flag);
3235                 if (ret) {
3236                         ret = i915_gem_evict_something(dev, vm, size, alignment,
3237                                                        obj->cache_level,
3238                                                        start, end,
3239                                                        flags);
3240                         if (ret == 0)
3241                                 goto search_free;
3242
3243                         goto err_free_vma;
3244                 }
3245         }
3246         if (WARN_ON(!i915_gem_valid_gtt_space(vma, obj->cache_level))) {
3247                 ret = -EINVAL;
3248                 goto err_remove_node;
3249         }
3250
3251         trace_i915_vma_bind(vma, flags);
3252         ret = i915_vma_bind(vma, obj->cache_level, flags);
3253         if (ret)
3254                 goto err_remove_node;
3255
3256         list_move_tail(&obj->global_list, &dev_priv->mm.bound_list);
3257         list_add_tail(&vma->vm_link, &vm->inactive_list);
3258
3259         return vma;
3260
3261 err_remove_node:
3262         drm_mm_remove_node(&vma->node);
3263 err_free_vma:
3264         i915_gem_vma_destroy(vma);
3265         vma = ERR_PTR(ret);
3266 err_unpin:
3267         i915_gem_object_unpin_pages(obj);
3268         return vma;
3269 }
3270
3271 bool
3272 i915_gem_clflush_object(struct drm_i915_gem_object *obj,
3273                         bool force)
3274 {
3275         /* If we don't have a page list set up, then we're not pinned
3276          * to GPU, and we can ignore the cache flush because it'll happen
3277          * again at bind time.
3278          */
3279         if (obj->pages == NULL)
3280                 return false;
3281
3282         /*
3283          * Stolen memory is always coherent with the GPU as it is explicitly
3284          * marked as wc by the system, or the system is cache-coherent.
3285          */
3286         if (obj->stolen || obj->phys_handle)
3287                 return false;
3288
3289         /* If the GPU is snooping the contents of the CPU cache,
3290          * we do not need to manually clear the CPU cache lines.  However,
3291          * the caches are only snooped when the render cache is
3292          * flushed/invalidated.  As we always have to emit invalidations
3293          * and flushes when moving into and out of the RENDER domain, correct
3294          * snooping behaviour occurs naturally as the result of our domain
3295          * tracking.
3296          */
3297         if (!force && cpu_cache_is_coherent(obj->base.dev, obj->cache_level)) {
3298                 obj->cache_dirty = true;
3299                 return false;
3300         }
3301
3302         trace_i915_gem_object_clflush(obj);
3303         drm_clflush_sg(obj->pages);
3304         obj->cache_dirty = false;
3305
3306         return true;
3307 }
3308
3309 /** Flushes the GTT write domain for the object if it's dirty. */
3310 static void
3311 i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
3312 {
3313         uint32_t old_write_domain;
3314
3315         if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
3316                 return;
3317
3318         /* No actual flushing is required for the GTT write domain.  Writes
3319          * to it immediately go to main memory as far as we know, so there's
3320          * no chipset flush.  It also doesn't land in render cache.
3321          *
3322          * However, we do have to enforce the order so that all writes through
3323          * the GTT land before any writes to the device, such as updates to
3324          * the GATT itself.
3325          */
3326         wmb();
3327
3328         old_write_domain = obj->base.write_domain;
3329         obj->base.write_domain = 0;
3330
3331         intel_fb_obj_flush(obj, false, ORIGIN_GTT);
3332
3333         trace_i915_gem_object_change_domain(obj,
3334                                             obj->base.read_domains,
3335                                             old_write_domain);
3336 }
3337
3338 /** Flushes the CPU write domain for the object if it's dirty. */
3339 static void
3340 i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
3341 {
3342         uint32_t old_write_domain;
3343
3344         if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
3345                 return;
3346
3347         if (i915_gem_clflush_object(obj, obj->pin_display))
3348                 i915_gem_chipset_flush(to_i915(obj->base.dev));
3349
3350         old_write_domain = obj->base.write_domain;
3351         obj->base.write_domain = 0;
3352
3353         intel_fb_obj_flush(obj, false, ORIGIN_CPU);
3354
3355         trace_i915_gem_object_change_domain(obj,
3356                                             obj->base.read_domains,
3357                                             old_write_domain);
3358 }
3359
3360 /**
3361  * Moves a single object to the GTT read, and possibly write domain.
3362  * @obj: object to act on
3363  * @write: ask for write access or read only
3364  *
3365  * This function returns when the move is complete, including waiting on
3366  * flushes to occur.
3367  */
3368 int
3369 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
3370 {
3371         struct drm_device *dev = obj->base.dev;
3372         struct drm_i915_private *dev_priv = to_i915(dev);
3373         struct i915_ggtt *ggtt = &dev_priv->ggtt;
3374         uint32_t old_write_domain, old_read_domains;
3375         struct i915_vma *vma;
3376         int ret;
3377
3378         ret = i915_gem_object_wait_rendering(obj, !write);
3379         if (ret)
3380                 return ret;
3381
3382         if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
3383                 return 0;
3384
3385         /* Flush and acquire obj->pages so that we are coherent through
3386          * direct access in memory with previous cached writes through
3387          * shmemfs and that our cache domain tracking remains valid.
3388          * For example, if the obj->filp was moved to swap without us
3389          * being notified and releasing the pages, we would mistakenly
3390          * continue to assume that the obj remained out of the CPU cached
3391          * domain.
3392          */
3393         ret = i915_gem_object_get_pages(obj);
3394         if (ret)
3395                 return ret;
3396
3397         i915_gem_object_flush_cpu_write_domain(obj);
3398
3399         /* Serialise direct access to this object with the barriers for
3400          * coherent writes from the GPU, by effectively invalidating the
3401          * GTT domain upon first access.
3402          */
3403         if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
3404                 mb();
3405
3406         old_write_domain = obj->base.write_domain;
3407         old_read_domains = obj->base.read_domains;
3408
3409         /* It should now be out of any other write domains, and we can update
3410          * the domain values for our changes.
3411          */
3412         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
3413         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
3414         if (write) {
3415                 obj->base.read_domains = I915_GEM_DOMAIN_GTT;
3416                 obj->base.write_domain = I915_GEM_DOMAIN_GTT;
3417                 obj->dirty = 1;
3418         }
3419
3420         trace_i915_gem_object_change_domain(obj,
3421                                             old_read_domains,
3422                                             old_write_domain);
3423
3424         /* And bump the LRU for this access */
3425         vma = i915_gem_obj_to_ggtt(obj);
3426         if (vma && drm_mm_node_allocated(&vma->node) && !obj->active)
3427                 list_move_tail(&vma->vm_link,
3428                                &ggtt->base.inactive_list);
3429
3430         return 0;
3431 }
3432
3433 /**
3434  * Changes the cache-level of an object across all VMA.
3435  * @obj: object to act on
3436  * @cache_level: new cache level to set for the object
3437  *
3438  * After this function returns, the object will be in the new cache-level
3439  * across all GTT and the contents of the backing storage will be coherent,
3440  * with respect to the new cache-level. In order to keep the backing storage
3441  * coherent for all users, we only allow a single cache level to be set
3442  * globally on the object and prevent it from being changed whilst the
3443  * hardware is reading from the object. That is if the object is currently
3444  * on the scanout it will be set to uncached (or equivalent display
3445  * cache coherency) and all non-MOCS GPU access will also be uncached so
3446  * that all direct access to the scanout remains coherent.
3447  */
3448 int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
3449                                     enum i915_cache_level cache_level)
3450 {
3451         struct drm_device *dev = obj->base.dev;
3452         struct i915_vma *vma, *next;
3453         bool bound = false;
3454         int ret = 0;
3455
3456         if (obj->cache_level == cache_level)
3457                 goto out;
3458
3459         /* Inspect the list of currently bound VMA and unbind any that would
3460          * be invalid given the new cache-level. This is principally to
3461          * catch the issue of the CS prefetch crossing page boundaries and
3462          * reading an invalid PTE on older architectures.
3463          */
3464         list_for_each_entry_safe(vma, next, &obj->vma_list, obj_link) {
3465                 if (!drm_mm_node_allocated(&vma->node))
3466                         continue;
3467
3468                 if (vma->pin_count) {
3469                         DRM_DEBUG("can not change the cache level of pinned objects\n");
3470                         return -EBUSY;
3471                 }
3472
3473                 if (!i915_gem_valid_gtt_space(vma, cache_level)) {
3474                         ret = i915_vma_unbind(vma);
3475                         if (ret)
3476                                 return ret;
3477                 } else
3478                         bound = true;
3479         }
3480
3481         /* We can reuse the existing drm_mm nodes but need to change the
3482          * cache-level on the PTE. We could simply unbind them all and
3483          * rebind with the correct cache-level on next use. However since
3484          * we already have a valid slot, dma mapping, pages etc, we may as
3485          * rewrite the PTE in the belief that doing so tramples upon less
3486          * state and so involves less work.
3487          */
3488         if (bound) {
3489                 /* Before we change the PTE, the GPU must not be accessing it.
3490                  * If we wait upon the object, we know that all the bound
3491                  * VMA are no longer active.
3492                  */
3493                 ret = i915_gem_object_wait_rendering(obj, false);
3494                 if (ret)
3495                         return ret;
3496
3497                 if (!HAS_LLC(dev) && cache_level != I915_CACHE_NONE) {
3498                         /* Access to snoopable pages through the GTT is
3499                          * incoherent and on some machines causes a hard
3500                          * lockup. Relinquish the CPU mmaping to force
3501                          * userspace to refault in the pages and we can
3502                          * then double check if the GTT mapping is still
3503                          * valid for that pointer access.
3504                          */
3505                         i915_gem_release_mmap(obj);
3506
3507                         /* As we no longer need a fence for GTT access,
3508                          * we can relinquish it now (and so prevent having
3509                          * to steal a fence from someone else on the next
3510                          * fence request). Note GPU activity would have
3511                          * dropped the fence as all snoopable access is
3512                          * supposed to be linear.
3513                          */
3514                         ret = i915_gem_object_put_fence(obj);
3515                         if (ret)
3516                                 return ret;
3517                 } else {
3518                         /* We either have incoherent backing store and
3519                          * so no GTT access or the architecture is fully
3520                          * coherent. In such cases, existing GTT mmaps
3521                          * ignore the cache bit in the PTE and we can
3522                          * rewrite it without confusing the GPU or having
3523                          * to force userspace to fault back in its mmaps.
3524                          */
3525                 }
3526
3527                 list_for_each_entry(vma, &obj->vma_list, obj_link) {
3528                         if (!drm_mm_node_allocated(&vma->node))
3529                                 continue;
3530
3531                         ret = i915_vma_bind(vma, cache_level, PIN_UPDATE);
3532                         if (ret)
3533                                 return ret;
3534                 }
3535         }
3536
3537         list_for_each_entry(vma, &obj->vma_list, obj_link)
3538                 vma->node.color = cache_level;
3539         obj->cache_level = cache_level;
3540
3541 out:
3542         /* Flush the dirty CPU caches to the backing storage so that the
3543          * object is now coherent at its new cache level (with respect
3544          * to the access domain).
3545          */
3546         if (obj->cache_dirty && cpu_write_needs_clflush(obj)) {
3547                 if (i915_gem_clflush_object(obj, true))
3548                         i915_gem_chipset_flush(to_i915(obj->base.dev));
3549         }
3550
3551         return 0;
3552 }
3553
3554 int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data,
3555                                struct drm_file *file)
3556 {
3557         struct drm_i915_gem_caching *args = data;
3558         struct drm_i915_gem_object *obj;
3559
3560         obj = i915_gem_object_lookup(file, args->handle);
3561         if (!obj)
3562                 return -ENOENT;
3563
3564         switch (obj->cache_level) {
3565         case I915_CACHE_LLC:
3566         case I915_CACHE_L3_LLC:
3567                 args->caching = I915_CACHING_CACHED;
3568                 break;
3569
3570         case I915_CACHE_WT:
3571                 args->caching = I915_CACHING_DISPLAY;
3572                 break;
3573
3574         default:
3575                 args->caching = I915_CACHING_NONE;
3576                 break;
3577         }
3578
3579         i915_gem_object_put_unlocked(obj);
3580         return 0;
3581 }
3582
3583 int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
3584                                struct drm_file *file)
3585 {
3586         struct drm_i915_private *dev_priv = to_i915(dev);
3587         struct drm_i915_gem_caching *args = data;
3588         struct drm_i915_gem_object *obj;
3589         enum i915_cache_level level;
3590         int ret;
3591
3592         switch (args->caching) {
3593         case I915_CACHING_NONE:
3594                 level = I915_CACHE_NONE;
3595                 break;
3596         case I915_CACHING_CACHED:
3597                 /*
3598                  * Due to a HW issue on BXT A stepping, GPU stores via a
3599                  * snooped mapping may leave stale data in a corresponding CPU
3600                  * cacheline, whereas normally such cachelines would get
3601                  * invalidated.
3602                  */
3603                 if (!HAS_LLC(dev) && !HAS_SNOOP(dev))
3604                         return -ENODEV;
3605
3606                 level = I915_CACHE_LLC;
3607                 break;
3608         case I915_CACHING_DISPLAY:
3609                 level = HAS_WT(dev) ? I915_CACHE_WT : I915_CACHE_NONE;
3610                 break;
3611         default:
3612                 return -EINVAL;
3613         }
3614
3615         intel_runtime_pm_get(dev_priv);
3616
3617         ret = i915_mutex_lock_interruptible(dev);
3618         if (ret)
3619                 goto rpm_put;
3620
3621         obj = i915_gem_object_lookup(file, args->handle);
3622         if (!obj) {
3623                 ret = -ENOENT;
3624                 goto unlock;
3625         }
3626
3627         ret = i915_gem_object_set_cache_level(obj, level);
3628
3629         i915_gem_object_put(obj);
3630 unlock:
3631         mutex_unlock(&dev->struct_mutex);
3632 rpm_put:
3633         intel_runtime_pm_put(dev_priv);
3634
3635         return ret;
3636 }
3637
3638 /*
3639  * Prepare buffer for display plane (scanout, cursors, etc).
3640  * Can be called from an uninterruptible phase (modesetting) and allows
3641  * any flushes to be pipelined (for pageflips).
3642  */
3643 int
3644 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
3645                                      u32 alignment,
3646                                      const struct i915_ggtt_view *view)
3647 {
3648         u32 old_read_domains, old_write_domain;
3649         int ret;
3650
3651         /* Mark the pin_display early so that we account for the
3652          * display coherency whilst setting up the cache domains.
3653          */
3654         obj->pin_display++;
3655
3656         /* The display engine is not coherent with the LLC cache on gen6.  As
3657          * a result, we make sure that the pinning that is about to occur is
3658          * done with uncached PTEs. This is lowest common denominator for all
3659          * chipsets.
3660          *
3661          * However for gen6+, we could do better by using the GFDT bit instead
3662          * of uncaching, which would allow us to flush all the LLC-cached data
3663          * with that bit in the PTE to main memory with just one PIPE_CONTROL.
3664          */
3665         ret = i915_gem_object_set_cache_level(obj,
3666                                               HAS_WT(obj->base.dev) ? I915_CACHE_WT : I915_CACHE_NONE);
3667         if (ret)
3668                 goto err_unpin_display;
3669
3670         /* As the user may map the buffer once pinned in the display plane
3671          * (e.g. libkms for the bootup splash), we have to ensure that we
3672          * always use map_and_fenceable for all scanout buffers.
3673          */
3674         ret = i915_gem_object_ggtt_pin(obj, view, alignment,
3675                                        view->type == I915_GGTT_VIEW_NORMAL ?
3676                                        PIN_MAPPABLE : 0);
3677         if (ret)
3678                 goto err_unpin_display;
3679
3680         i915_gem_object_flush_cpu_write_domain(obj);
3681
3682         old_write_domain = obj->base.write_domain;
3683         old_read_domains = obj->base.read_domains;
3684
3685         /* It should now be out of any other write domains, and we can update
3686          * the domain values for our changes.
3687          */
3688         obj->base.write_domain = 0;
3689         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
3690
3691         trace_i915_gem_object_change_domain(obj,
3692                                             old_read_domains,
3693                                             old_write_domain);
3694
3695         return 0;
3696
3697 err_unpin_display:
3698         obj->pin_display--;
3699         return ret;
3700 }
3701
3702 void
3703 i915_gem_object_unpin_from_display_plane(struct drm_i915_gem_object *obj,
3704                                          const struct i915_ggtt_view *view)
3705 {
3706         if (WARN_ON(obj->pin_display == 0))
3707                 return;
3708
3709         i915_gem_object_ggtt_unpin_view(obj, view);
3710
3711         obj->pin_display--;
3712 }
3713
3714 /**
3715  * Moves a single object to the CPU read, and possibly write domain.
3716  * @obj: object to act on
3717  * @write: requesting write or read-only access
3718  *
3719  * This function returns when the move is complete, including waiting on
3720  * flushes to occur.
3721  */
3722 int
3723 i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
3724 {
3725         uint32_t old_write_domain, old_read_domains;
3726         int ret;
3727
3728         ret = i915_gem_object_wait_rendering(obj, !write);
3729         if (ret)
3730                 return ret;
3731
3732         if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
3733                 return 0;
3734
3735         i915_gem_object_flush_gtt_write_domain(obj);
3736
3737         old_write_domain = obj->base.write_domain;
3738         old_read_domains = obj->base.read_domains;
3739
3740         /* Flush the CPU cache if it's still invalid. */
3741         if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
3742                 i915_gem_clflush_object(obj, false);
3743
3744                 obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
3745         }
3746
3747         /* It should now be out of any other write domains, and we can update
3748          * the domain values for our changes.
3749          */
3750         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
3751
3752         /* If we're writing through the CPU, then the GPU read domains will
3753          * need to be invalidated at next use.
3754          */
3755         if (write) {
3756                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
3757                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
3758         }
3759
3760         trace_i915_gem_object_change_domain(obj,
3761                                             old_read_domains,
3762                                             old_write_domain);
3763
3764         return 0;
3765 }
3766
3767 /* Throttle our rendering by waiting until the ring has completed our requests
3768  * emitted over 20 msec ago.
3769  *
3770  * Note that if we were to use the current jiffies each time around the loop,
3771  * we wouldn't escape the function with any frames outstanding if the time to
3772  * render a frame was over 20ms.
3773  *
3774  * This should get us reasonable parallelism between CPU and GPU but also
3775  * relatively low latency when blocking on a particular request to finish.
3776  */
3777 static int
3778 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
3779 {
3780         struct drm_i915_private *dev_priv = to_i915(dev);
3781         struct drm_i915_file_private *file_priv = file->driver_priv;
3782         unsigned long recent_enough = jiffies - DRM_I915_THROTTLE_JIFFIES;
3783         struct drm_i915_gem_request *request, *target = NULL;
3784         int ret;
3785
3786         ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
3787         if (ret)
3788                 return ret;
3789
3790         /* ABI: return -EIO if already wedged */
3791         if (i915_terminally_wedged(&dev_priv->gpu_error))
3792                 return -EIO;
3793
3794         spin_lock(&file_priv->mm.lock);
3795         list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
3796                 if (time_after_eq(request->emitted_jiffies, recent_enough))
3797                         break;
3798
3799                 /*
3800                  * Note that the request might not have been submitted yet.
3801                  * In which case emitted_jiffies will be zero.
3802                  */
3803                 if (!request->emitted_jiffies)
3804                         continue;
3805
3806                 target = request;
3807         }
3808         if (target)
3809                 i915_gem_request_get(target);
3810         spin_unlock(&file_priv->mm.lock);
3811
3812         if (target == NULL)
3813                 return 0;
3814
3815         ret = __i915_wait_request(target, true, NULL, NULL);
3816         i915_gem_request_put(target);
3817
3818         return ret;
3819 }
3820
3821 static bool
3822 i915_vma_misplaced(struct i915_vma *vma, uint32_t alignment, uint64_t flags)
3823 {
3824         struct drm_i915_gem_object *obj = vma->obj;
3825
3826         if (alignment &&
3827             vma->node.start & (alignment - 1))
3828                 return true;
3829
3830         if (flags & PIN_MAPPABLE && !obj->map_and_fenceable)
3831                 return true;
3832
3833         if (flags & PIN_OFFSET_BIAS &&
3834             vma->node.start < (flags & PIN_OFFSET_MASK))
3835                 return true;
3836
3837         if (flags & PIN_OFFSET_FIXED &&
3838             vma->node.start != (flags & PIN_OFFSET_MASK))
3839                 return true;
3840
3841         return false;
3842 }
3843
3844 void __i915_vma_set_map_and_fenceable(struct i915_vma *vma)
3845 {
3846         struct drm_i915_gem_object *obj = vma->obj;
3847         bool mappable, fenceable;
3848         u32 fence_size, fence_alignment;
3849
3850         fence_size = i915_gem_get_gtt_size(obj->base.dev,
3851                                            obj->base.size,
3852                                            obj->tiling_mode);
3853         fence_alignment = i915_gem_get_gtt_alignment(obj->base.dev,
3854                                                      obj->base.size,
3855                                                      obj->tiling_mode,
3856                                                      true);
3857
3858         fenceable = (vma->node.size == fence_size &&
3859                      (vma->node.start & (fence_alignment - 1)) == 0);
3860
3861         mappable = (vma->node.start + fence_size <=
3862                     to_i915(obj->base.dev)->ggtt.mappable_end);
3863
3864         obj->map_and_fenceable = mappable && fenceable;
3865 }
3866
3867 static int
3868 i915_gem_object_do_pin(struct drm_i915_gem_object *obj,
3869                        struct i915_address_space *vm,
3870                        const struct i915_ggtt_view *ggtt_view,
3871                        uint32_t alignment,
3872                        uint64_t flags)
3873 {
3874         struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
3875         struct i915_vma *vma;
3876         unsigned bound;
3877         int ret;
3878
3879         if (WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base))
3880                 return -ENODEV;
3881
3882         if (WARN_ON(flags & (PIN_GLOBAL | PIN_MAPPABLE) && !i915_is_ggtt(vm)))
3883                 return -EINVAL;
3884
3885         if (WARN_ON((flags & (PIN_MAPPABLE | PIN_GLOBAL)) == PIN_MAPPABLE))
3886                 return -EINVAL;
3887
3888         if (WARN_ON(i915_is_ggtt(vm) != !!ggtt_view))
3889                 return -EINVAL;
3890
3891         vma = ggtt_view ? i915_gem_obj_to_ggtt_view(obj, ggtt_view) :
3892                           i915_gem_obj_to_vma(obj, vm);
3893
3894         if (vma) {
3895                 if (WARN_ON(vma->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT))
3896                         return -EBUSY;
3897
3898                 if (i915_vma_misplaced(vma, alignment, flags)) {
3899                         WARN(vma->pin_count,
3900                              "bo is already pinned in %s with incorrect alignment:"
3901                              " offset=%08x %08x, req.alignment=%x, req.map_and_fenceable=%d,"
3902                              " obj->map_and_fenceable=%d\n",
3903                              ggtt_view ? "ggtt" : "ppgtt",
3904                              upper_32_bits(vma->node.start),
3905                              lower_32_bits(vma->node.start),
3906                              alignment,
3907                              !!(flags & PIN_MAPPABLE),
3908                              obj->map_and_fenceable);
3909                         ret = i915_vma_unbind(vma);
3910                         if (ret)
3911                                 return ret;
3912
3913                         vma = NULL;
3914                 }
3915         }
3916
3917         bound = vma ? vma->bound : 0;
3918         if (vma == NULL || !drm_mm_node_allocated(&vma->node)) {
3919                 vma = i915_gem_object_bind_to_vm(obj, vm, ggtt_view, alignment,
3920                                                  flags);
3921                 if (IS_ERR(vma))
3922                         return PTR_ERR(vma);
3923         } else {
3924                 ret = i915_vma_bind(vma, obj->cache_level, flags);
3925                 if (ret)
3926                         return ret;
3927         }
3928
3929         if (ggtt_view && ggtt_view->type == I915_GGTT_VIEW_NORMAL &&
3930             (bound ^ vma->bound) & GLOBAL_BIND) {
3931                 __i915_vma_set_map_and_fenceable(vma);
3932                 WARN_ON(flags & PIN_MAPPABLE && !obj->map_and_fenceable);
3933         }
3934
3935         vma->pin_count++;
3936         return 0;
3937 }
3938
3939 int
3940 i915_gem_object_pin(struct drm_i915_gem_object *obj,
3941                     struct i915_address_space *vm,
3942                     uint32_t alignment,
3943                     uint64_t flags)
3944 {
3945         return i915_gem_object_do_pin(obj, vm,
3946                                       i915_is_ggtt(vm) ? &i915_ggtt_view_normal : NULL,
3947                                       alignment, flags);
3948 }
3949
3950 int
3951 i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
3952                          const struct i915_ggtt_view *view,
3953                          uint32_t alignment,
3954                          uint64_t flags)
3955 {
3956         struct drm_device *dev = obj->base.dev;
3957         struct drm_i915_private *dev_priv = to_i915(dev);
3958         struct i915_ggtt *ggtt = &dev_priv->ggtt;
3959
3960         BUG_ON(!view);
3961
3962         return i915_gem_object_do_pin(obj, &ggtt->base, view,
3963                                       alignment, flags | PIN_GLOBAL);
3964 }
3965
3966 void
3967 i915_gem_object_ggtt_unpin_view(struct drm_i915_gem_object *obj,
3968                                 const struct i915_ggtt_view *view)
3969 {
3970         struct i915_vma *vma = i915_gem_obj_to_ggtt_view(obj, view);
3971
3972         WARN_ON(vma->pin_count == 0);
3973         WARN_ON(!i915_gem_obj_ggtt_bound_view(obj, view));
3974
3975         --vma->pin_count;
3976 }
3977
3978 int
3979 i915_gem_busy_ioctl(struct drm_device *dev, void *data,
3980                     struct drm_file *file)
3981 {
3982         struct drm_i915_gem_busy *args = data;
3983         struct drm_i915_gem_object *obj;
3984         int ret;
3985
3986         ret = i915_mutex_lock_interruptible(dev);
3987         if (ret)
3988                 return ret;
3989
3990         obj = i915_gem_object_lookup(file, args->handle);
3991         if (!obj) {
3992                 ret = -ENOENT;
3993                 goto unlock;
3994         }
3995
3996         /* Count all active objects as busy, even if they are currently not used
3997          * by the gpu. Users of this interface expect objects to eventually
3998          * become non-busy without any further actions, therefore emit any
3999          * necessary flushes here.
4000          */
4001         ret = i915_gem_object_flush_active(obj);
4002         if (ret)
4003                 goto unref;
4004
4005         args->busy = 0;
4006         if (obj->active) {
4007                 int i;
4008
4009                 for (i = 0; i < I915_NUM_ENGINES; i++) {
4010                         struct drm_i915_gem_request *req;
4011
4012                         req = obj->last_read_req[i];
4013                         if (req)
4014                                 args->busy |= 1 << (16 + req->engine->exec_id);
4015                 }
4016                 if (obj->last_write_req)
4017                         args->busy |= obj->last_write_req->engine->exec_id;
4018         }
4019
4020 unref:
4021         i915_gem_object_put(obj);
4022 unlock:
4023         mutex_unlock(&dev->struct_mutex);
4024         return ret;
4025 }
4026
4027 int
4028 i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
4029                         struct drm_file *file_priv)
4030 {
4031         return i915_gem_ring_throttle(dev, file_priv);
4032 }
4033
4034 int
4035 i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
4036                        struct drm_file *file_priv)
4037 {
4038         struct drm_i915_private *dev_priv = to_i915(dev);
4039         struct drm_i915_gem_madvise *args = data;
4040         struct drm_i915_gem_object *obj;
4041         int ret;
4042
4043         switch (args->madv) {
4044         case I915_MADV_DONTNEED:
4045         case I915_MADV_WILLNEED:
4046             break;
4047         default:
4048             return -EINVAL;
4049         }
4050
4051         ret = i915_mutex_lock_interruptible(dev);
4052         if (ret)
4053                 return ret;
4054
4055         obj = i915_gem_object_lookup(file_priv, args->handle);
4056         if (!obj) {
4057                 ret = -ENOENT;
4058                 goto unlock;
4059         }
4060
4061         if (i915_gem_obj_is_pinned(obj)) {
4062                 ret = -EINVAL;
4063                 goto out;
4064         }
4065
4066         if (obj->pages &&
4067             obj->tiling_mode != I915_TILING_NONE &&
4068             dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
4069                 if (obj->madv == I915_MADV_WILLNEED)
4070                         i915_gem_object_unpin_pages(obj);
4071                 if (args->madv == I915_MADV_WILLNEED)
4072                         i915_gem_object_pin_pages(obj);
4073         }
4074
4075         if (obj->madv != __I915_MADV_PURGED)
4076                 obj->madv = args->madv;
4077
4078         /* if the object is no longer attached, discard its backing storage */
4079         if (obj->madv == I915_MADV_DONTNEED && obj->pages == NULL)
4080                 i915_gem_object_truncate(obj);
4081
4082         args->retained = obj->madv != __I915_MADV_PURGED;
4083
4084 out:
4085         i915_gem_object_put(obj);
4086 unlock:
4087         mutex_unlock(&dev->struct_mutex);
4088         return ret;
4089 }
4090
4091 void i915_gem_object_init(struct drm_i915_gem_object *obj,
4092                           const struct drm_i915_gem_object_ops *ops)
4093 {
4094         int i;
4095
4096         INIT_LIST_HEAD(&obj->global_list);
4097         for (i = 0; i < I915_NUM_ENGINES; i++)
4098                 INIT_LIST_HEAD(&obj->engine_list[i]);
4099         INIT_LIST_HEAD(&obj->obj_exec_link);
4100         INIT_LIST_HEAD(&obj->vma_list);
4101         INIT_LIST_HEAD(&obj->batch_pool_link);
4102
4103         obj->ops = ops;
4104
4105         obj->fence_reg = I915_FENCE_REG_NONE;
4106         obj->madv = I915_MADV_WILLNEED;
4107
4108         i915_gem_info_add_obj(to_i915(obj->base.dev), obj->base.size);
4109 }
4110
4111 static const struct drm_i915_gem_object_ops i915_gem_object_ops = {
4112         .flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE,
4113         .get_pages = i915_gem_object_get_pages_gtt,
4114         .put_pages = i915_gem_object_put_pages_gtt,
4115 };
4116
4117 struct drm_i915_gem_object *i915_gem_object_create(struct drm_device *dev,
4118                                                   size_t size)
4119 {
4120         struct drm_i915_gem_object *obj;
4121         struct address_space *mapping;
4122         gfp_t mask;
4123         int ret;
4124
4125         obj = i915_gem_object_alloc(dev);
4126         if (obj == NULL)
4127                 return ERR_PTR(-ENOMEM);
4128
4129         ret = drm_gem_object_init(dev, &obj->base, size);
4130         if (ret)
4131                 goto fail;
4132
4133         mask = GFP_HIGHUSER | __GFP_RECLAIMABLE;
4134         if (IS_CRESTLINE(dev) || IS_BROADWATER(dev)) {
4135                 /* 965gm cannot relocate objects above 4GiB. */
4136                 mask &= ~__GFP_HIGHMEM;
4137                 mask |= __GFP_DMA32;
4138         }
4139
4140         mapping = file_inode(obj->base.filp)->i_mapping;
4141         mapping_set_gfp_mask(mapping, mask);
4142
4143         i915_gem_object_init(obj, &i915_gem_object_ops);
4144
4145         obj->base.write_domain = I915_GEM_DOMAIN_CPU;
4146         obj->base.read_domains = I915_GEM_DOMAIN_CPU;
4147
4148         if (HAS_LLC(dev)) {
4149                 /* On some devices, we can have the GPU use the LLC (the CPU
4150                  * cache) for about a 10% performance improvement
4151                  * compared to uncached.  Graphics requests other than
4152                  * display scanout are coherent with the CPU in
4153                  * accessing this cache.  This means in this mode we
4154                  * don't need to clflush on the CPU side, and on the
4155                  * GPU side we only need to flush internal caches to
4156                  * get data visible to the CPU.
4157                  *
4158                  * However, we maintain the display planes as UC, and so
4159                  * need to rebind when first used as such.
4160                  */
4161                 obj->cache_level = I915_CACHE_LLC;
4162         } else
4163                 obj->cache_level = I915_CACHE_NONE;
4164
4165         trace_i915_gem_object_create(obj);
4166
4167         return obj;
4168
4169 fail:
4170         i915_gem_object_free(obj);
4171
4172         return ERR_PTR(ret);
4173 }
4174
4175 static bool discard_backing_storage(struct drm_i915_gem_object *obj)
4176 {
4177         /* If we are the last user of the backing storage (be it shmemfs
4178          * pages or stolen etc), we know that the pages are going to be
4179          * immediately released. In this case, we can then skip copying
4180          * back the contents from the GPU.
4181          */
4182
4183         if (obj->madv != I915_MADV_WILLNEED)
4184                 return false;
4185
4186         if (obj->base.filp == NULL)
4187                 return true;
4188
4189         /* At first glance, this looks racy, but then again so would be
4190          * userspace racing mmap against close. However, the first external
4191          * reference to the filp can only be obtained through the
4192          * i915_gem_mmap_ioctl() which safeguards us against the user
4193          * acquiring such a reference whilst we are in the middle of
4194          * freeing the object.
4195          */
4196         return atomic_long_read(&obj->base.filp->f_count) == 1;
4197 }
4198
4199 void i915_gem_free_object(struct drm_gem_object *gem_obj)
4200 {
4201         struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
4202         struct drm_device *dev = obj->base.dev;
4203         struct drm_i915_private *dev_priv = to_i915(dev);
4204         struct i915_vma *vma, *next;
4205
4206         intel_runtime_pm_get(dev_priv);
4207
4208         trace_i915_gem_object_destroy(obj);
4209
4210         list_for_each_entry_safe(vma, next, &obj->vma_list, obj_link) {
4211                 int ret;
4212
4213                 vma->pin_count = 0;
4214                 ret = __i915_vma_unbind_no_wait(vma);
4215                 if (WARN_ON(ret == -ERESTARTSYS)) {
4216                         bool was_interruptible;
4217
4218                         was_interruptible = dev_priv->mm.interruptible;
4219                         dev_priv->mm.interruptible = false;
4220
4221                         WARN_ON(i915_vma_unbind(vma));
4222
4223                         dev_priv->mm.interruptible = was_interruptible;
4224                 }
4225         }
4226
4227         /* Stolen objects don't hold a ref, but do hold pin count. Fix that up
4228          * before progressing. */
4229         if (obj->stolen)
4230                 i915_gem_object_unpin_pages(obj);
4231
4232         WARN_ON(obj->frontbuffer_bits);
4233
4234         if (obj->pages && obj->madv == I915_MADV_WILLNEED &&
4235             dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES &&
4236             obj->tiling_mode != I915_TILING_NONE)
4237                 i915_gem_object_unpin_pages(obj);
4238
4239         if (WARN_ON(obj->pages_pin_count))
4240                 obj->pages_pin_count = 0;
4241         if (discard_backing_storage(obj))
4242                 obj->madv = I915_MADV_DONTNEED;
4243         i915_gem_object_put_pages(obj);
4244
4245         BUG_ON(obj->pages);
4246
4247         if (obj->base.import_attach)
4248                 drm_prime_gem_destroy(&obj->base, NULL);
4249
4250         if (obj->ops->release)
4251                 obj->ops->release(obj);
4252
4253         drm_gem_object_release(&obj->base);
4254         i915_gem_info_remove_obj(dev_priv, obj->base.size);
4255
4256         kfree(obj->bit_17);
4257         i915_gem_object_free(obj);
4258
4259         intel_runtime_pm_put(dev_priv);
4260 }
4261
4262 struct i915_vma *i915_gem_obj_to_vma(struct drm_i915_gem_object *obj,
4263                                      struct i915_address_space *vm)
4264 {
4265         struct i915_vma *vma;
4266         list_for_each_entry(vma, &obj->vma_list, obj_link) {
4267                 if (vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL &&
4268                     vma->vm == vm)
4269                         return vma;
4270         }
4271         return NULL;
4272 }
4273
4274 struct i915_vma *i915_gem_obj_to_ggtt_view(struct drm_i915_gem_object *obj,
4275                                            const struct i915_ggtt_view *view)
4276 {
4277         struct i915_vma *vma;
4278
4279         GEM_BUG_ON(!view);
4280
4281         list_for_each_entry(vma, &obj->vma_list, obj_link)
4282                 if (vma->is_ggtt && i915_ggtt_view_equal(&vma->ggtt_view, view))
4283                         return vma;
4284         return NULL;
4285 }
4286
4287 void i915_gem_vma_destroy(struct i915_vma *vma)
4288 {
4289         WARN_ON(vma->node.allocated);
4290
4291         /* Keep the vma as a placeholder in the execbuffer reservation lists */
4292         if (!list_empty(&vma->exec_list))
4293                 return;
4294
4295         if (!vma->is_ggtt)
4296                 i915_ppgtt_put(i915_vm_to_ppgtt(vma->vm));
4297
4298         list_del(&vma->obj_link);
4299
4300         kmem_cache_free(to_i915(vma->obj->base.dev)->vmas, vma);
4301 }
4302
4303 static void
4304 i915_gem_stop_engines(struct drm_device *dev)
4305 {
4306         struct drm_i915_private *dev_priv = to_i915(dev);
4307         struct intel_engine_cs *engine;
4308
4309         for_each_engine(engine, dev_priv)
4310                 dev_priv->gt.stop_engine(engine);
4311 }
4312
4313 int
4314 i915_gem_suspend(struct drm_device *dev)
4315 {
4316         struct drm_i915_private *dev_priv = to_i915(dev);
4317         int ret = 0;
4318
4319         intel_suspend_gt_powersave(dev_priv);
4320
4321         mutex_lock(&dev->struct_mutex);
4322
4323         /* We have to flush all the executing contexts to main memory so
4324          * that they can saved in the hibernation image. To ensure the last
4325          * context image is coherent, we have to switch away from it. That
4326          * leaves the dev_priv->kernel_context still active when
4327          * we actually suspend, and its image in memory may not match the GPU
4328          * state. Fortunately, the kernel_context is disposable and we do
4329          * not rely on its state.
4330          */
4331         ret = i915_gem_switch_to_kernel_context(dev_priv);
4332         if (ret)
4333                 goto err;
4334
4335         ret = i915_gem_wait_for_idle(dev_priv);
4336         if (ret)
4337                 goto err;
4338
4339         i915_gem_retire_requests(dev_priv);
4340
4341         /* Note that rather than stopping the engines, all we have to do
4342          * is assert that every RING_HEAD == RING_TAIL (all execution complete)
4343          * and similar for all logical context images (to ensure they are
4344          * all ready for hibernation).
4345          */
4346         i915_gem_stop_engines(dev);
4347         i915_gem_context_lost(dev_priv);
4348         mutex_unlock(&dev->struct_mutex);
4349
4350         cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work);
4351         cancel_delayed_work_sync(&dev_priv->gt.retire_work);
4352         flush_delayed_work(&dev_priv->gt.idle_work);
4353
4354         /* Assert that we sucessfully flushed all the work and
4355          * reset the GPU back to its idle, low power state.
4356          */
4357         WARN_ON(dev_priv->gt.awake);
4358
4359         return 0;
4360
4361 err:
4362         mutex_unlock(&dev->struct_mutex);
4363         return ret;
4364 }
4365
4366 void i915_gem_resume(struct drm_device *dev)
4367 {
4368         struct drm_i915_private *dev_priv = to_i915(dev);
4369
4370         mutex_lock(&dev->struct_mutex);
4371         i915_gem_restore_gtt_mappings(dev);
4372
4373         /* As we didn't flush the kernel context before suspend, we cannot
4374          * guarantee that the context image is complete. So let's just reset
4375          * it and start again.
4376          */
4377         if (i915.enable_execlists)
4378                 intel_lr_context_reset(dev_priv, dev_priv->kernel_context);
4379
4380         mutex_unlock(&dev->struct_mutex);
4381 }
4382
4383 void i915_gem_init_swizzling(struct drm_device *dev)
4384 {
4385         struct drm_i915_private *dev_priv = to_i915(dev);
4386
4387         if (INTEL_INFO(dev)->gen < 5 ||
4388             dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
4389                 return;
4390
4391         I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
4392                                  DISP_TILE_SURFACE_SWIZZLING);
4393
4394         if (IS_GEN5(dev))
4395                 return;
4396
4397         I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL);
4398         if (IS_GEN6(dev))
4399                 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
4400         else if (IS_GEN7(dev))
4401                 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
4402         else if (IS_GEN8(dev))
4403                 I915_WRITE(GAMTARBMODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_BDW));
4404         else
4405                 BUG();
4406 }
4407
4408 static void init_unused_ring(struct drm_device *dev, u32 base)
4409 {
4410         struct drm_i915_private *dev_priv = to_i915(dev);
4411
4412         I915_WRITE(RING_CTL(base), 0);
4413         I915_WRITE(RING_HEAD(base), 0);
4414         I915_WRITE(RING_TAIL(base), 0);
4415         I915_WRITE(RING_START(base), 0);
4416 }
4417
4418 static void init_unused_rings(struct drm_device *dev)
4419 {
4420         if (IS_I830(dev)) {
4421                 init_unused_ring(dev, PRB1_BASE);
4422                 init_unused_ring(dev, SRB0_BASE);
4423                 init_unused_ring(dev, SRB1_BASE);
4424                 init_unused_ring(dev, SRB2_BASE);
4425                 init_unused_ring(dev, SRB3_BASE);
4426         } else if (IS_GEN2(dev)) {
4427                 init_unused_ring(dev, SRB0_BASE);
4428                 init_unused_ring(dev, SRB1_BASE);
4429         } else if (IS_GEN3(dev)) {
4430                 init_unused_ring(dev, PRB1_BASE);
4431                 init_unused_ring(dev, PRB2_BASE);
4432         }
4433 }
4434
4435 int
4436 i915_gem_init_hw(struct drm_device *dev)
4437 {
4438         struct drm_i915_private *dev_priv = to_i915(dev);
4439         struct intel_engine_cs *engine;
4440         int ret;
4441
4442         /* Double layer security blanket, see i915_gem_init() */
4443         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
4444
4445         if (HAS_EDRAM(dev) && INTEL_GEN(dev_priv) < 9)
4446                 I915_WRITE(HSW_IDICR, I915_READ(HSW_IDICR) | IDIHASHMSK(0xf));
4447
4448         if (IS_HASWELL(dev))
4449                 I915_WRITE(MI_PREDICATE_RESULT_2, IS_HSW_GT3(dev) ?
4450                            LOWER_SLICE_ENABLED : LOWER_SLICE_DISABLED);
4451
4452         if (HAS_PCH_NOP(dev)) {
4453                 if (IS_IVYBRIDGE(dev)) {
4454                         u32 temp = I915_READ(GEN7_MSG_CTL);
4455                         temp &= ~(WAIT_FOR_PCH_FLR_ACK | WAIT_FOR_PCH_RESET_ACK);
4456                         I915_WRITE(GEN7_MSG_CTL, temp);
4457                 } else if (INTEL_INFO(dev)->gen >= 7) {
4458                         u32 temp = I915_READ(HSW_NDE_RSTWRN_OPT);
4459                         temp &= ~RESET_PCH_HANDSHAKE_ENABLE;
4460                         I915_WRITE(HSW_NDE_RSTWRN_OPT, temp);
4461                 }
4462         }
4463
4464         i915_gem_init_swizzling(dev);
4465
4466         /*
4467          * At least 830 can leave some of the unused rings
4468          * "active" (ie. head != tail) after resume which
4469          * will prevent c3 entry. Makes sure all unused rings
4470          * are totally idle.
4471          */
4472         init_unused_rings(dev);
4473
4474         BUG_ON(!dev_priv->kernel_context);
4475
4476         ret = i915_ppgtt_init_hw(dev);
4477         if (ret) {
4478                 DRM_ERROR("PPGTT enable HW failed %d\n", ret);
4479                 goto out;
4480         }
4481
4482         /* Need to do basic initialisation of all rings first: */
4483         for_each_engine(engine, dev_priv) {
4484                 ret = engine->init_hw(engine);
4485                 if (ret)
4486                         goto out;
4487         }
4488
4489         intel_mocs_init_l3cc_table(dev);
4490
4491         /* We can't enable contexts until all firmware is loaded */
4492         ret = intel_guc_setup(dev);
4493         if (ret)
4494                 goto out;
4495
4496 out:
4497         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
4498         return ret;
4499 }
4500
4501 bool intel_sanitize_semaphores(struct drm_i915_private *dev_priv, int value)
4502 {
4503         if (INTEL_INFO(dev_priv)->gen < 6)
4504                 return false;
4505
4506         /* TODO: make semaphores and Execlists play nicely together */
4507         if (i915.enable_execlists)
4508                 return false;
4509
4510         if (value >= 0)
4511                 return value;
4512
4513 #ifdef CONFIG_INTEL_IOMMU
4514         /* Enable semaphores on SNB when IO remapping is off */
4515         if (INTEL_INFO(dev_priv)->gen == 6 && intel_iommu_gfx_mapped)
4516                 return false;
4517 #endif
4518
4519         return true;
4520 }
4521
4522 int i915_gem_init(struct drm_device *dev)
4523 {
4524         struct drm_i915_private *dev_priv = to_i915(dev);
4525         int ret;
4526
4527         mutex_lock(&dev->struct_mutex);
4528
4529         if (!i915.enable_execlists) {
4530                 dev_priv->gt.cleanup_engine = intel_engine_cleanup;
4531                 dev_priv->gt.stop_engine = intel_engine_stop;
4532         } else {
4533                 dev_priv->gt.cleanup_engine = intel_logical_ring_cleanup;
4534                 dev_priv->gt.stop_engine = intel_logical_ring_stop;
4535         }
4536
4537         /* This is just a security blanket to placate dragons.
4538          * On some systems, we very sporadically observe that the first TLBs
4539          * used by the CS may be stale, despite us poking the TLB reset. If
4540          * we hold the forcewake during initialisation these problems
4541          * just magically go away.
4542          */
4543         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
4544
4545         i915_gem_init_userptr(dev_priv);
4546         i915_gem_init_ggtt(dev);
4547
4548         ret = i915_gem_context_init(dev);
4549         if (ret)
4550                 goto out_unlock;
4551
4552         ret = intel_engines_init(dev);
4553         if (ret)
4554                 goto out_unlock;
4555
4556         ret = i915_gem_init_hw(dev);
4557         if (ret == -EIO) {
4558                 /* Allow engine initialisation to fail by marking the GPU as
4559                  * wedged. But we only want to do this where the GPU is angry,
4560                  * for all other failure, such as an allocation failure, bail.
4561                  */
4562                 DRM_ERROR("Failed to initialize GPU, declaring it wedged\n");
4563                 atomic_or(I915_WEDGED, &dev_priv->gpu_error.reset_counter);
4564                 ret = 0;
4565         }
4566
4567 out_unlock:
4568         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
4569         mutex_unlock(&dev->struct_mutex);
4570
4571         return ret;
4572 }
4573
4574 void
4575 i915_gem_cleanup_engines(struct drm_device *dev)
4576 {
4577         struct drm_i915_private *dev_priv = to_i915(dev);
4578         struct intel_engine_cs *engine;
4579
4580         for_each_engine(engine, dev_priv)
4581                 dev_priv->gt.cleanup_engine(engine);
4582 }
4583
4584 static void
4585 init_engine_lists(struct intel_engine_cs *engine)
4586 {
4587         INIT_LIST_HEAD(&engine->active_list);
4588         INIT_LIST_HEAD(&engine->request_list);
4589 }
4590
4591 void
4592 i915_gem_load_init_fences(struct drm_i915_private *dev_priv)
4593 {
4594         struct drm_device *dev = &dev_priv->drm;
4595
4596         if (INTEL_INFO(dev_priv)->gen >= 7 && !IS_VALLEYVIEW(dev_priv) &&
4597             !IS_CHERRYVIEW(dev_priv))
4598                 dev_priv->num_fence_regs = 32;
4599         else if (INTEL_INFO(dev_priv)->gen >= 4 || IS_I945G(dev_priv) ||
4600                  IS_I945GM(dev_priv) || IS_G33(dev_priv))
4601                 dev_priv->num_fence_regs = 16;
4602         else
4603                 dev_priv->num_fence_regs = 8;
4604
4605         if (intel_vgpu_active(dev_priv))
4606                 dev_priv->num_fence_regs =
4607                                 I915_READ(vgtif_reg(avail_rs.fence_num));
4608
4609         /* Initialize fence registers to zero */
4610         i915_gem_restore_fences(dev);
4611
4612         i915_gem_detect_bit_6_swizzle(dev);
4613 }
4614
4615 void
4616 i915_gem_load_init(struct drm_device *dev)
4617 {
4618         struct drm_i915_private *dev_priv = to_i915(dev);
4619         int i;
4620
4621         dev_priv->objects =
4622                 kmem_cache_create("i915_gem_object",
4623                                   sizeof(struct drm_i915_gem_object), 0,
4624                                   SLAB_HWCACHE_ALIGN,
4625                                   NULL);
4626         dev_priv->vmas =
4627                 kmem_cache_create("i915_gem_vma",
4628                                   sizeof(struct i915_vma), 0,
4629                                   SLAB_HWCACHE_ALIGN,
4630                                   NULL);
4631         dev_priv->requests =
4632                 kmem_cache_create("i915_gem_request",
4633                                   sizeof(struct drm_i915_gem_request), 0,
4634                                   SLAB_HWCACHE_ALIGN,
4635                                   NULL);
4636
4637         INIT_LIST_HEAD(&dev_priv->vm_list);
4638         INIT_LIST_HEAD(&dev_priv->context_list);
4639         INIT_LIST_HEAD(&dev_priv->mm.unbound_list);
4640         INIT_LIST_HEAD(&dev_priv->mm.bound_list);
4641         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
4642         for (i = 0; i < I915_NUM_ENGINES; i++)
4643                 init_engine_lists(&dev_priv->engine[i]);
4644         for (i = 0; i < I915_MAX_NUM_FENCES; i++)
4645                 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
4646         INIT_DELAYED_WORK(&dev_priv->gt.retire_work,
4647                           i915_gem_retire_work_handler);
4648         INIT_DELAYED_WORK(&dev_priv->gt.idle_work,
4649                           i915_gem_idle_work_handler);
4650         init_waitqueue_head(&dev_priv->gpu_error.wait_queue);
4651         init_waitqueue_head(&dev_priv->gpu_error.reset_queue);
4652
4653         dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
4654
4655         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
4656
4657         init_waitqueue_head(&dev_priv->pending_flip_queue);
4658
4659         dev_priv->mm.interruptible = true;
4660
4661         mutex_init(&dev_priv->fb_tracking.lock);
4662 }
4663
4664 void i915_gem_load_cleanup(struct drm_device *dev)
4665 {
4666         struct drm_i915_private *dev_priv = to_i915(dev);
4667
4668         kmem_cache_destroy(dev_priv->requests);
4669         kmem_cache_destroy(dev_priv->vmas);
4670         kmem_cache_destroy(dev_priv->objects);
4671 }
4672
4673 int i915_gem_freeze_late(struct drm_i915_private *dev_priv)
4674 {
4675         struct drm_i915_gem_object *obj;
4676
4677         /* Called just before we write the hibernation image.
4678          *
4679          * We need to update the domain tracking to reflect that the CPU
4680          * will be accessing all the pages to create and restore from the
4681          * hibernation, and so upon restoration those pages will be in the
4682          * CPU domain.
4683          *
4684          * To make sure the hibernation image contains the latest state,
4685          * we update that state just before writing out the image.
4686          */
4687
4688         list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list) {
4689                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
4690                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
4691         }
4692
4693         list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
4694                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
4695                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
4696         }
4697
4698         return 0;
4699 }
4700
4701 void i915_gem_release(struct drm_device *dev, struct drm_file *file)
4702 {
4703         struct drm_i915_file_private *file_priv = file->driver_priv;
4704         struct drm_i915_gem_request *request;
4705
4706         /* Clean up our request list when the client is going away, so that
4707          * later retire_requests won't dereference our soon-to-be-gone
4708          * file_priv.
4709          */
4710         spin_lock(&file_priv->mm.lock);
4711         list_for_each_entry(request, &file_priv->mm.request_list, client_list)
4712                 request->file_priv = NULL;
4713         spin_unlock(&file_priv->mm.lock);
4714
4715         if (!list_empty(&file_priv->rps.link)) {
4716                 spin_lock(&to_i915(dev)->rps.client_lock);
4717                 list_del(&file_priv->rps.link);
4718                 spin_unlock(&to_i915(dev)->rps.client_lock);
4719         }
4720 }
4721
4722 int i915_gem_open(struct drm_device *dev, struct drm_file *file)
4723 {
4724         struct drm_i915_file_private *file_priv;
4725         int ret;
4726
4727         DRM_DEBUG_DRIVER("\n");
4728
4729         file_priv = kzalloc(sizeof(*file_priv), GFP_KERNEL);
4730         if (!file_priv)
4731                 return -ENOMEM;
4732
4733         file->driver_priv = file_priv;
4734         file_priv->dev_priv = to_i915(dev);
4735         file_priv->file = file;
4736         INIT_LIST_HEAD(&file_priv->rps.link);
4737
4738         spin_lock_init(&file_priv->mm.lock);
4739         INIT_LIST_HEAD(&file_priv->mm.request_list);
4740
4741         file_priv->bsd_engine = -1;
4742
4743         ret = i915_gem_context_open(dev, file);
4744         if (ret)
4745                 kfree(file_priv);
4746
4747         return ret;
4748 }
4749
4750 /**
4751  * i915_gem_track_fb - update frontbuffer tracking
4752  * @old: current GEM buffer for the frontbuffer slots
4753  * @new: new GEM buffer for the frontbuffer slots
4754  * @frontbuffer_bits: bitmask of frontbuffer slots
4755  *
4756  * This updates the frontbuffer tracking bits @frontbuffer_bits by clearing them
4757  * from @old and setting them in @new. Both @old and @new can be NULL.
4758  */
4759 void i915_gem_track_fb(struct drm_i915_gem_object *old,
4760                        struct drm_i915_gem_object *new,
4761                        unsigned frontbuffer_bits)
4762 {
4763         if (old) {
4764                 WARN_ON(!mutex_is_locked(&old->base.dev->struct_mutex));
4765                 WARN_ON(!(old->frontbuffer_bits & frontbuffer_bits));
4766                 old->frontbuffer_bits &= ~frontbuffer_bits;
4767         }
4768
4769         if (new) {
4770                 WARN_ON(!mutex_is_locked(&new->base.dev->struct_mutex));
4771                 WARN_ON(new->frontbuffer_bits & frontbuffer_bits);
4772                 new->frontbuffer_bits |= frontbuffer_bits;
4773         }
4774 }
4775
4776 /* All the new VM stuff */
4777 u64 i915_gem_obj_offset(struct drm_i915_gem_object *o,
4778                         struct i915_address_space *vm)
4779 {
4780         struct drm_i915_private *dev_priv = to_i915(o->base.dev);
4781         struct i915_vma *vma;
4782
4783         WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base);
4784
4785         list_for_each_entry(vma, &o->vma_list, obj_link) {
4786                 if (vma->is_ggtt &&
4787                     vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL)
4788                         continue;
4789                 if (vma->vm == vm)
4790                         return vma->node.start;
4791         }
4792
4793         WARN(1, "%s vma for this object not found.\n",
4794              i915_is_ggtt(vm) ? "global" : "ppgtt");
4795         return -1;
4796 }
4797
4798 u64 i915_gem_obj_ggtt_offset_view(struct drm_i915_gem_object *o,
4799                                   const struct i915_ggtt_view *view)
4800 {
4801         struct i915_vma *vma;
4802
4803         list_for_each_entry(vma, &o->vma_list, obj_link)
4804                 if (vma->is_ggtt && i915_ggtt_view_equal(&vma->ggtt_view, view))
4805                         return vma->node.start;
4806
4807         WARN(1, "global vma for this object not found. (view=%u)\n", view->type);
4808         return -1;
4809 }
4810
4811 bool i915_gem_obj_bound(struct drm_i915_gem_object *o,
4812                         struct i915_address_space *vm)
4813 {
4814         struct i915_vma *vma;
4815
4816         list_for_each_entry(vma, &o->vma_list, obj_link) {
4817                 if (vma->is_ggtt &&
4818                     vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL)
4819                         continue;
4820                 if (vma->vm == vm && drm_mm_node_allocated(&vma->node))
4821                         return true;
4822         }
4823
4824         return false;
4825 }
4826
4827 bool i915_gem_obj_ggtt_bound_view(struct drm_i915_gem_object *o,
4828                                   const struct i915_ggtt_view *view)
4829 {
4830         struct i915_vma *vma;
4831
4832         list_for_each_entry(vma, &o->vma_list, obj_link)
4833                 if (vma->is_ggtt &&
4834                     i915_ggtt_view_equal(&vma->ggtt_view, view) &&
4835                     drm_mm_node_allocated(&vma->node))
4836                         return true;
4837
4838         return false;
4839 }
4840
4841 bool i915_gem_obj_bound_any(struct drm_i915_gem_object *o)
4842 {
4843         struct i915_vma *vma;
4844
4845         list_for_each_entry(vma, &o->vma_list, obj_link)
4846                 if (drm_mm_node_allocated(&vma->node))
4847                         return true;
4848
4849         return false;
4850 }
4851
4852 unsigned long i915_gem_obj_ggtt_size(struct drm_i915_gem_object *o)
4853 {
4854         struct i915_vma *vma;
4855
4856         GEM_BUG_ON(list_empty(&o->vma_list));
4857
4858         list_for_each_entry(vma, &o->vma_list, obj_link) {
4859                 if (vma->is_ggtt &&
4860                     vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL)
4861                         return vma->node.size;
4862         }
4863
4864         return 0;
4865 }
4866
4867 bool i915_gem_obj_is_pinned(struct drm_i915_gem_object *obj)
4868 {
4869         struct i915_vma *vma;
4870         list_for_each_entry(vma, &obj->vma_list, obj_link)
4871                 if (vma->pin_count > 0)
4872                         return true;
4873
4874         return false;
4875 }
4876
4877 /* Like i915_gem_object_get_page(), but mark the returned page dirty */
4878 struct page *
4879 i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj, int n)
4880 {
4881         struct page *page;
4882
4883         /* Only default objects have per-page dirty tracking */
4884         if (WARN_ON(!i915_gem_object_has_struct_page(obj)))
4885                 return NULL;
4886
4887         page = i915_gem_object_get_page(obj, n);
4888         set_page_dirty(page);
4889         return page;
4890 }
4891
4892 /* Allocate a new GEM object and fill it with the supplied data */
4893 struct drm_i915_gem_object *
4894 i915_gem_object_create_from_data(struct drm_device *dev,
4895                                  const void *data, size_t size)
4896 {
4897         struct drm_i915_gem_object *obj;
4898         struct sg_table *sg;
4899         size_t bytes;
4900         int ret;
4901
4902         obj = i915_gem_object_create(dev, round_up(size, PAGE_SIZE));
4903         if (IS_ERR(obj))
4904                 return obj;
4905
4906         ret = i915_gem_object_set_to_cpu_domain(obj, true);
4907         if (ret)
4908                 goto fail;
4909
4910         ret = i915_gem_object_get_pages(obj);
4911         if (ret)
4912                 goto fail;
4913
4914         i915_gem_object_pin_pages(obj);
4915         sg = obj->pages;
4916         bytes = sg_copy_from_buffer(sg->sgl, sg->nents, (void *)data, size);
4917         obj->dirty = 1;         /* Backing store is now out of date */
4918         i915_gem_object_unpin_pages(obj);
4919
4920         if (WARN_ON(bytes != size)) {
4921                 DRM_ERROR("Incomplete copy, wrote %zu of %zu", bytes, size);
4922                 ret = -EFAULT;
4923                 goto fail;
4924         }
4925
4926         return obj;
4927
4928 fail:
4929         i915_gem_object_put(obj);
4930         return ERR_PTR(ret);
4931 }