drivers/gpu/drm/i915/i915_gem.c

   1 /*
   2  * Copyright © 2008-2015 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <[email protected]>
  25  *
  26  */
  27
  28 #include <drm/drmP.h>
  29 #include <drm/drm_vma_manager.h>
  30 #include <drm/i915_drm.h>
  31 #include "i915_drv.h"
  32 #include "i915_gem_dmabuf.h"
  33 #include "i915_vgpu.h"
  34 #include "i915_trace.h"
  35 #include "intel_drv.h"
  36 #include "intel_frontbuffer.h"
  37 #include "intel_mocs.h"
  38 #include <linux/reservation.h>
  39 #include <linux/shmem_fs.h>
  40 #include <linux/slab.h>
  41 #include <linux/swap.h>
  42 #include <linux/pci.h>
  43 #include <linux/dma-buf.h>
  44
  45 static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
  46 static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
  47
  48 static bool cpu_cache_is_coherent(struct drm_device *dev,
  49                                   enum i915_cache_level level)
  50 {
  51         return HAS_LLC(dev) || level != I915_CACHE_NONE;
  52 }
  53
  54 static bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj)
  55 {
  56         if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
  57                 return false;
  58
  59         if (!cpu_cache_is_coherent(obj->base.dev, obj->cache_level))
  60                 return true;
  61
  62         return obj->pin_display;
  63 }
  64
  65 static int
  66 insert_mappable_node(struct drm_i915_private *i915,
  67                      struct drm_mm_node *node, u32 size)
  68 {
  69         memset(node, 0, sizeof(*node));
  70         return drm_mm_insert_node_in_range_generic(&i915->ggtt.base.mm, node,
  71                                                    size, 0, 0, 0,
  72                                                    i915->ggtt.mappable_end,
  73                                                    DRM_MM_SEARCH_DEFAULT,
  74                                                    DRM_MM_CREATE_DEFAULT);
  75 }
  76
  77 static void
  78 remove_mappable_node(struct drm_mm_node *node)
  79 {
  80         drm_mm_remove_node(node);
  81 }
  82
  83 /* some bookkeeping */
  84 static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
  85                                   size_t size)
  86 {
  87         spin_lock(&dev_priv->mm.object_stat_lock);
  88         dev_priv->mm.object_count++;
  89         dev_priv->mm.object_memory += size;
  90         spin_unlock(&dev_priv->mm.object_stat_lock);
  91 }
  92
  93 static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
  94                                      size_t size)
  95 {
  96         spin_lock(&dev_priv->mm.object_stat_lock);
  97         dev_priv->mm.object_count--;
  98         dev_priv->mm.object_memory -= size;
  99         spin_unlock(&dev_priv->mm.object_stat_lock);
 100 }
 101
 102 static int
 103 i915_gem_wait_for_error(struct i915_gpu_error *error)
 104 {
 105         int ret;
 106
 107         if (!i915_reset_in_progress(error))
 108                 return 0;
 109
 110         /*
 111          * Only wait 10 seconds for the gpu reset to complete to avoid hanging
 112          * userspace. If it takes that long something really bad is going on and
 113          * we should simply try to bail out and fail as gracefully as possible.
 114          */
 115         ret = wait_event_interruptible_timeout(error->reset_queue,
 116                                                !i915_reset_in_progress(error),
 117                                                10*HZ);
 118         if (ret == 0) {
 119                 DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
 120                 return -EIO;
 121         } else if (ret < 0) {
 122                 return ret;
 123         } else {
 124                 return 0;
 125         }
 126 }
 127
 128 int i915_mutex_lock_interruptible(struct drm_device *dev)
 129 {
 130         struct drm_i915_private *dev_priv = to_i915(dev);
 131         int ret;
 132
 133         ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
 134         if (ret)
 135                 return ret;
 136
 137         ret = mutex_lock_interruptible(&dev->struct_mutex);
 138         if (ret)
 139                 return ret;
 140
 141         return 0;
 142 }
 143
 144 int
 145 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
 146                             struct drm_file *file)
 147 {
 148         struct drm_i915_private *dev_priv = to_i915(dev);
 149         struct i915_ggtt *ggtt = &dev_priv->ggtt;
 150         struct drm_i915_gem_get_aperture *args = data;
 151         struct i915_vma *vma;
 152         size_t pinned;
 153
 154         pinned = 0;
 155         mutex_lock(&dev->struct_mutex);
 156         list_for_each_entry(vma, &ggtt->base.active_list, vm_link)
 157                 if (i915_vma_is_pinned(vma))
 158                         pinned += vma->node.size;
 159         list_for_each_entry(vma, &ggtt->base.inactive_list, vm_link)
 160                 if (i915_vma_is_pinned(vma))
 161                         pinned += vma->node.size;
 162         mutex_unlock(&dev->struct_mutex);
 163
 164         args->aper_size = ggtt->base.total;
 165         args->aper_available_size = args->aper_size - pinned;
 166
 167         return 0;
 168 }
 169
 170 static int
 171 i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj)
 172 {
 173         struct address_space *mapping = file_inode(obj->base.filp)->i_mapping;
 174         char *vaddr = obj->phys_handle->vaddr;
 175         struct sg_table *st;
 176         struct scatterlist *sg;
 177         int i;
 178
 179         if (WARN_ON(i915_gem_object_needs_bit17_swizzle(obj)))
 180                 return -EINVAL;
 181
 182         for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
 183                 struct page *page;
 184                 char *src;
 185
 186                 page = shmem_read_mapping_page(mapping, i);
 187                 if (IS_ERR(page))
 188                         return PTR_ERR(page);
 189
 190                 src = kmap_atomic(page);
 191                 memcpy(vaddr, src, PAGE_SIZE);
 192                 drm_clflush_virt_range(vaddr, PAGE_SIZE);
 193                 kunmap_atomic(src);
 194
 195                 put_page(page);
 196                 vaddr += PAGE_SIZE;
 197         }
 198
 199         i915_gem_chipset_flush(to_i915(obj->base.dev));
 200
 201         st = kmalloc(sizeof(*st), GFP_KERNEL);
 202         if (st == NULL)
 203                 return -ENOMEM;
 204
 205         if (sg_alloc_table(st, 1, GFP_KERNEL)) {
 206                 kfree(st);
 207                 return -ENOMEM;
 208         }
 209
 210         sg = st->sgl;
 211         sg->offset = 0;
 212         sg->length = obj->base.size;
 213
 214         sg_dma_address(sg) = obj->phys_handle->busaddr;
 215         sg_dma_len(sg) = obj->base.size;
 216
 217         obj->pages = st;
 218         return 0;
 219 }
 220
 221 static void
 222 i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj)
 223 {
 224         int ret;
 225
 226         BUG_ON(obj->madv == __I915_MADV_PURGED);
 227
 228         ret = i915_gem_object_set_to_cpu_domain(obj, true);
 229         if (WARN_ON(ret)) {
 230                 /* In the event of a disaster, abandon all caches and
 231                  * hope for the best.
 232                  */
 233                 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
 234         }
 235
 236         if (obj->madv == I915_MADV_DONTNEED)
 237                 obj->dirty = 0;
 238
 239         if (obj->dirty) {
 240                 struct address_space *mapping = file_inode(obj->base.filp)->i_mapping;
 241                 char *vaddr = obj->phys_handle->vaddr;
 242                 int i;
 243
 244                 for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
 245                         struct page *page;
 246                         char *dst;
 247
 248                         page = shmem_read_mapping_page(mapping, i);
 249                         if (IS_ERR(page))
 250                                 continue;
 251
 252                         dst = kmap_atomic(page);
 253                         drm_clflush_virt_range(vaddr, PAGE_SIZE);
 254                         memcpy(dst, vaddr, PAGE_SIZE);
 255                         kunmap_atomic(dst);
 256
 257                         set_page_dirty(page);
 258                         if (obj->madv == I915_MADV_WILLNEED)
 259                                 mark_page_accessed(page);
 260                         put_page(page);
 261                         vaddr += PAGE_SIZE;
 262                 }
 263                 obj->dirty = 0;
 264         }
 265
 266         sg_free_table(obj->pages);
 267         kfree(obj->pages);
 268 }
 269
 270 static void
 271 i915_gem_object_release_phys(struct drm_i915_gem_object *obj)
 272 {
 273         drm_pci_free(obj->base.dev, obj->phys_handle);
 274 }
 275
 276 static const struct drm_i915_gem_object_ops i915_gem_phys_ops = {
 277         .get_pages = i915_gem_object_get_pages_phys,
 278         .put_pages = i915_gem_object_put_pages_phys,
 279         .release = i915_gem_object_release_phys,
 280 };
 281
 282 int i915_gem_object_unbind(struct drm_i915_gem_object *obj)
 283 {
 284         struct i915_vma *vma;
 285         LIST_HEAD(still_in_list);
 286         int ret;
 287
 288         lockdep_assert_held(&obj->base.dev->struct_mutex);
 289
 290         /* Closed vma are removed from the obj->vma_list - but they may
 291          * still have an active binding on the object. To remove those we
 292          * must wait for all rendering to complete to the object (as unbinding
 293          * must anyway), and retire the requests.
 294          */
 295         ret = i915_gem_object_wait_rendering(obj, false);
 296         if (ret)
 297                 return ret;
 298
 299         i915_gem_retire_requests(to_i915(obj->base.dev));
 300
 301         while ((vma = list_first_entry_or_null(&obj->vma_list,
 302                                                struct i915_vma,
 303                                                obj_link))) {
 304                 list_move_tail(&vma->obj_link, &still_in_list);
 305                 ret = i915_vma_unbind(vma);
 306                 if (ret)
 307                         break;
 308         }
 309         list_splice(&still_in_list, &obj->vma_list);
 310
 311         return ret;
 312 }
 313
 314 /**
 315  * Ensures that all rendering to the object has completed and the object is
 316  * safe to unbind from the GTT or access from the CPU.
 317  * @obj: i915 gem object
 318  * @readonly: waiting for just read access or read-write access
 319  */
 320 int
 321 i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
 322                                bool readonly)
 323 {
 324         struct reservation_object *resv;
 325         struct i915_gem_active *active;
 326         unsigned long active_mask;
 327         int idx;
 328
 329         lockdep_assert_held(&obj->base.dev->struct_mutex);
 330
 331         if (!readonly) {
 332                 active = obj->last_read;
 333                 active_mask = i915_gem_object_get_active(obj);
 334         } else {
 335                 active_mask = 1;
 336                 active = &obj->last_write;
 337         }
 338
 339         for_each_active(active_mask, idx) {
 340                 int ret;
 341
 342                 ret = i915_gem_active_wait(&active[idx],
 343                                            &obj->base.dev->struct_mutex);
 344                 if (ret)
 345                         return ret;
 346         }
 347
 348         resv = i915_gem_object_get_dmabuf_resv(obj);
 349         if (resv) {
 350                 long err;
 351
 352                 err = reservation_object_wait_timeout_rcu(resv, !readonly, true,
 353                                                           MAX_SCHEDULE_TIMEOUT);
 354                 if (err < 0)
 355                         return err;
 356         }
 357
 358         return 0;
 359 }
 360
 361 /* A nonblocking variant of the above wait. Must be called prior to
 362  * acquiring the mutex for the object, as the object state may change
 363  * during this call. A reference must be held by the caller for the object.
 364  */
 365 static __must_check int
 366 __unsafe_wait_rendering(struct drm_i915_gem_object *obj,
 367                         struct intel_rps_client *rps,
 368                         bool readonly)
 369 {
 370         struct i915_gem_active *active;
 371         unsigned long active_mask;
 372         int idx;
 373
 374         active_mask = __I915_BO_ACTIVE(obj);
 375         if (!active_mask)
 376                 return 0;
 377
 378         if (!readonly) {
 379                 active = obj->last_read;
 380         } else {
 381                 active_mask = 1;
 382                 active = &obj->last_write;
 383         }
 384
 385         for_each_active(active_mask, idx) {
 386                 int ret;
 387
 388                 ret = i915_gem_active_wait_unlocked(&active[idx],
 389                                                     true, NULL, rps);
 390                 if (ret)
 391                         return ret;
 392         }
 393
 394         return 0;
 395 }
 396
 397 static struct intel_rps_client *to_rps_client(struct drm_file *file)
 398 {
 399         struct drm_i915_file_private *fpriv = file->driver_priv;
 400
 401         return &fpriv->rps;
 402 }
 403
 404 int
 405 i915_gem_object_attach_phys(struct drm_i915_gem_object *obj,
 406                             int align)
 407 {
 408         drm_dma_handle_t *phys;
 409         int ret;
 410
 411         if (obj->phys_handle) {
 412                 if ((unsigned long)obj->phys_handle->vaddr & (align -1))
 413                         return -EBUSY;
 414
 415                 return 0;
 416         }
 417
 418         if (obj->madv != I915_MADV_WILLNEED)
 419                 return -EFAULT;
 420
 421         if (obj->base.filp == NULL)
 422                 return -EINVAL;
 423
 424         ret = i915_gem_object_unbind(obj);
 425         if (ret)
 426                 return ret;
 427
 428         ret = i915_gem_object_put_pages(obj);
 429         if (ret)
 430                 return ret;
 431
 432         /* create a new object */
 433         phys = drm_pci_alloc(obj->base.dev, obj->base.size, align);
 434         if (!phys)
 435                 return -ENOMEM;
 436
 437         obj->phys_handle = phys;
 438         obj->ops = &i915_gem_phys_ops;
 439
 440         return i915_gem_object_get_pages(obj);
 441 }
 442
 443 static int
 444 i915_gem_phys_pwrite(struct drm_i915_gem_object *obj,
 445                      struct drm_i915_gem_pwrite *args,
 446                      struct drm_file *file_priv)
 447 {
 448         struct drm_device *dev = obj->base.dev;
 449         void *vaddr = obj->phys_handle->vaddr + args->offset;
 450         char __user *user_data = u64_to_user_ptr(args->data_ptr);
 451         int ret = 0;
 452
 453         /* We manually control the domain here and pretend that it
 454          * remains coherent i.e. in the GTT domain, like shmem_pwrite.
 455          */
 456         ret = i915_gem_object_wait_rendering(obj, false);
 457         if (ret)
 458                 return ret;
 459
 460         intel_fb_obj_invalidate(obj, ORIGIN_CPU);
 461         if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) {
 462                 unsigned long unwritten;
 463
 464                 /* The physical object once assigned is fixed for the lifetime
 465                  * of the obj, so we can safely drop the lock and continue
 466                  * to access vaddr.
 467                  */
 468                 mutex_unlock(&dev->struct_mutex);
 469                 unwritten = copy_from_user(vaddr, user_data, args->size);
 470                 mutex_lock(&dev->struct_mutex);
 471                 if (unwritten) {
 472                         ret = -EFAULT;
 473                         goto out;
 474                 }
 475         }
 476
 477         drm_clflush_virt_range(vaddr, args->size);
 478         i915_gem_chipset_flush(to_i915(dev));
 479
 480 out:
 481         intel_fb_obj_flush(obj, false, ORIGIN_CPU);
 482         return ret;
 483 }
 484
 485 void *i915_gem_object_alloc(struct drm_device *dev)
 486 {
 487         struct drm_i915_private *dev_priv = to_i915(dev);
 488         return kmem_cache_zalloc(dev_priv->objects, GFP_KERNEL);
 489 }
 490
 491 void i915_gem_object_free(struct drm_i915_gem_object *obj)
 492 {
 493         struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
 494         kmem_cache_free(dev_priv->objects, obj);
 495 }
 496
 497 static int
 498 i915_gem_create(struct drm_file *file,
 499                 struct drm_device *dev,
 500                 uint64_t size,
 501                 uint32_t *handle_p)
 502 {
 503         struct drm_i915_gem_object *obj;
 504         int ret;
 505         u32 handle;
 506
 507         size = roundup(size, PAGE_SIZE);
 508         if (size == 0)
 509                 return -EINVAL;
 510
 511         /* Allocate the new object */
 512         obj = i915_gem_object_create(dev, size);
 513         if (IS_ERR(obj))
 514                 return PTR_ERR(obj);
 515
 516         ret = drm_gem_handle_create(file, &obj->base, &handle);
 517         /* drop reference from allocate - handle holds it now */
 518         i915_gem_object_put_unlocked(obj);
 519         if (ret)
 520                 return ret;
 521
 522         *handle_p = handle;
 523         return 0;
 524 }
 525
 526 int
 527 i915_gem_dumb_create(struct drm_file *file,
 528                      struct drm_device *dev,
 529                      struct drm_mode_create_dumb *args)
 530 {
 531         /* have to work out size/pitch and return them */
 532         args->pitch = ALIGN(args->width * DIV_ROUND_UP(args->bpp, 8), 64);
 533         args->size = args->pitch * args->height;
 534         return i915_gem_create(file, dev,
 535                                args->size, &args->handle);
 536 }
 537
 538 /**
 539  * Creates a new mm object and returns a handle to it.
 540  * @dev: drm device pointer
 541  * @data: ioctl data blob
 542  * @file: drm file pointer
 543  */
 544 int
 545 i915_gem_create_ioctl(struct drm_device *dev, void *data,
 546                       struct drm_file *file)
 547 {
 548         struct drm_i915_gem_create *args = data;
 549
 550         return i915_gem_create(file, dev,
 551                                args->size, &args->handle);
 552 }
 553
 554 static inline int
 555 __copy_to_user_swizzled(char __user *cpu_vaddr,
 556                         const char *gpu_vaddr, int gpu_offset,
 557                         int length)
 558 {
 559         int ret, cpu_offset = 0;
 560
 561         while (length > 0) {
 562                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
 563                 int this_length = min(cacheline_end - gpu_offset, length);
 564                 int swizzled_gpu_offset = gpu_offset ^ 64;
 565
 566                 ret = __copy_to_user(cpu_vaddr + cpu_offset,
 567                                      gpu_vaddr + swizzled_gpu_offset,
 568                                      this_length);
 569                 if (ret)
 570                         return ret + length;
 571
 572                 cpu_offset += this_length;
 573                 gpu_offset += this_length;
 574                 length -= this_length;
 575         }
 576
 577         return 0;
 578 }
 579
 580 static inline int
 581 __copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
 582                           const char __user *cpu_vaddr,
 583                           int length)
 584 {
 585         int ret, cpu_offset = 0;
 586
 587         while (length > 0) {
 588                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
 589                 int this_length = min(cacheline_end - gpu_offset, length);
 590                 int swizzled_gpu_offset = gpu_offset ^ 64;
 591
 592                 ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
 593                                        cpu_vaddr + cpu_offset,
 594                                        this_length);
 595                 if (ret)
 596                         return ret + length;
 597
 598                 cpu_offset += this_length;
 599                 gpu_offset += this_length;
 600                 length -= this_length;
 601         }
 602
 603         return 0;
 604 }
 605
 606 /*
 607  * Pins the specified object's pages and synchronizes the object with
 608  * GPU accesses. Sets needs_clflush to non-zero if the caller should
 609  * flush the object from the CPU cache.
 610  */
 611 int i915_gem_obj_prepare_shmem_read(struct drm_i915_gem_object *obj,
 612                                     int *needs_clflush)
 613 {
 614         int ret;
 615
 616         *needs_clflush = 0;
 617
 618         if (WARN_ON(!i915_gem_object_has_struct_page(obj)))
 619                 return -EINVAL;
 620
 621         ret = i915_gem_object_wait_rendering(obj, true);
 622         if (ret)
 623                 return ret;
 624
 625         if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) {
 626                 /* If we're not in the cpu read domain, set ourself into the gtt
 627                  * read domain and manually flush cachelines (if required). This
 628                  * optimizes for the case when the gpu will dirty the data
 629                  * anyway again before the next pread happens. */
 630                 *needs_clflush = !cpu_cache_is_coherent(obj->base.dev,
 631                                                         obj->cache_level);
 632         }
 633
 634         ret = i915_gem_object_get_pages(obj);
 635         if (ret)
 636                 return ret;
 637
 638         i915_gem_object_pin_pages(obj);
 639
 640         return ret;
 641 }
 642
 643 /* Per-page copy function for the shmem pread fastpath.
 644  * Flushes invalid cachelines before reading the target if
 645  * needs_clflush is set. */
 646 static int
 647 shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length,
 648                  char __user *user_data,
 649                  bool page_do_bit17_swizzling, bool needs_clflush)
 650 {
 651         char *vaddr;
 652         int ret;
 653
 654         if (unlikely(page_do_bit17_swizzling))
 655                 return -EINVAL;
 656
 657         vaddr = kmap_atomic(page);
 658         if (needs_clflush)
 659                 drm_clflush_virt_range(vaddr + shmem_page_offset,
 660                                        page_length);
 661         ret = __copy_to_user_inatomic(user_data,
 662                                       vaddr + shmem_page_offset,
 663                                       page_length);
 664         kunmap_atomic(vaddr);
 665
 666         return ret ? -EFAULT : 0;
 667 }
 668
 669 static void
 670 shmem_clflush_swizzled_range(char *addr, unsigned long length,
 671                              bool swizzled)
 672 {
 673         if (unlikely(swizzled)) {
 674                 unsigned long start = (unsigned long) addr;
 675                 unsigned long end = (unsigned long) addr + length;
 676
 677                 /* For swizzling simply ensure that we always flush both
 678                  * channels. Lame, but simple and it works. Swizzled
 679                  * pwrite/pread is far from a hotpath - current userspace
 680                  * doesn't use it at all. */
 681                 start = round_down(start, 128);
 682                 end = round_up(end, 128);
 683
 684                 drm_clflush_virt_range((void *)start, end - start);
 685         } else {
 686                 drm_clflush_virt_range(addr, length);
 687         }
 688
 689 }
 690
 691 /* Only difference to the fast-path function is that this can handle bit17
 692  * and uses non-atomic copy and kmap functions. */
 693 static int
 694 shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
 695                  char __user *user_data,
 696                  bool page_do_bit17_swizzling, bool needs_clflush)
 697 {
 698         char *vaddr;
 699         int ret;
 700
 701         vaddr = kmap(page);
 702         if (needs_clflush)
 703                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
 704                                              page_length,
 705                                              page_do_bit17_swizzling);
 706
 707         if (page_do_bit17_swizzling)
 708                 ret = __copy_to_user_swizzled(user_data,
 709                                               vaddr, shmem_page_offset,
 710                                               page_length);
 711         else
 712                 ret = __copy_to_user(user_data,
 713                                      vaddr + shmem_page_offset,
 714                                      page_length);
 715         kunmap(page);
 716
 717         return ret ? - EFAULT : 0;
 718 }
 719
 720 static inline unsigned long
 721 slow_user_access(struct io_mapping *mapping,
 722                  uint64_t page_base, int page_offset,
 723                  char __user *user_data,
 724                  unsigned long length, bool pwrite)
 725 {
 726         void __iomem *ioaddr;
 727         void *vaddr;
 728         uint64_t unwritten;
 729
 730         ioaddr = io_mapping_map_wc(mapping, page_base, PAGE_SIZE);
 731         /* We can use the cpu mem copy function because this is X86. */
 732         vaddr = (void __force *)ioaddr + page_offset;
 733         if (pwrite)
 734                 unwritten = __copy_from_user(vaddr, user_data, length);
 735         else
 736                 unwritten = __copy_to_user(user_data, vaddr, length);
 737
 738         io_mapping_unmap(ioaddr);
 739         return unwritten;
 740 }
 741
 742 static int
 743 i915_gem_gtt_pread(struct drm_device *dev,
 744                    struct drm_i915_gem_object *obj, uint64_t size,
 745                    uint64_t data_offset, uint64_t data_ptr)
 746 {
 747         struct drm_i915_private *dev_priv = to_i915(dev);
 748         struct i915_ggtt *ggtt = &dev_priv->ggtt;
 749         struct drm_mm_node node;
 750         char __user *user_data;
 751         uint64_t remain;
 752         uint64_t offset;
 753         int ret;
 754
 755         ret = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, PIN_MAPPABLE);
 756         if (ret) {
 757                 ret = insert_mappable_node(dev_priv, &node, PAGE_SIZE);
 758                 if (ret)
 759                         goto out;
 760
 761                 ret = i915_gem_object_get_pages(obj);
 762                 if (ret) {
 763                         remove_mappable_node(&node);
 764                         goto out;
 765                 }
 766
 767                 i915_gem_object_pin_pages(obj);
 768         } else {
 769                 node.start = i915_gem_obj_ggtt_offset(obj);
 770                 node.allocated = false;
 771                 ret = i915_gem_object_put_fence(obj);
 772                 if (ret)
 773                         goto out_unpin;
 774         }
 775
 776         ret = i915_gem_object_set_to_gtt_domain(obj, false);
 777         if (ret)
 778                 goto out_unpin;
 779
 780         user_data = u64_to_user_ptr(data_ptr);
 781         remain = size;
 782         offset = data_offset;
 783
 784         mutex_unlock(&dev->struct_mutex);
 785         if (likely(!i915.prefault_disable)) {
 786                 ret = fault_in_multipages_writeable(user_data, remain);
 787                 if (ret) {
 788                         mutex_lock(&dev->struct_mutex);
 789                         goto out_unpin;
 790                 }
 791         }
 792
 793         while (remain > 0) {
 794                 /* Operation in this page
 795                  *
 796                  * page_base = page offset within aperture
 797                  * page_offset = offset within page
 798                  * page_length = bytes to copy for this page
 799                  */
 800                 u32 page_base = node.start;
 801                 unsigned page_offset = offset_in_page(offset);
 802                 unsigned page_length = PAGE_SIZE - page_offset;
 803                 page_length = remain < page_length ? remain : page_length;
 804                 if (node.allocated) {
 805                         wmb();
 806                         ggtt->base.insert_page(&ggtt->base,
 807                                                i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT),
 808                                                node.start,
 809                                                I915_CACHE_NONE, 0);
 810                         wmb();
 811                 } else {
 812                         page_base += offset & PAGE_MASK;
 813                 }
 814                 /* This is a slow read/write as it tries to read from
 815                  * and write to user memory which may result into page
 816                  * faults, and so we cannot perform this under struct_mutex.
 817                  */
 818                 if (slow_user_access(ggtt->mappable, page_base,
 819                                      page_offset, user_data,
 820                                      page_length, false)) {
 821                         ret = -EFAULT;
 822                         break;
 823                 }
 824
 825                 remain -= page_length;
 826                 user_data += page_length;
 827                 offset += page_length;
 828         }
 829
 830         mutex_lock(&dev->struct_mutex);
 831         if (ret == 0 && (obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0) {
 832                 /* The user has modified the object whilst we tried
 833                  * reading from it, and we now have no idea what domain
 834                  * the pages should be in. As we have just been touching
 835                  * them directly, flush everything back to the GTT
 836                  * domain.
 837                  */
 838                 ret = i915_gem_object_set_to_gtt_domain(obj, false);
 839         }
 840
 841 out_unpin:
 842         if (node.allocated) {
 843                 wmb();
 844                 ggtt->base.clear_range(&ggtt->base,
 845                                        node.start, node.size,
 846                                        true);
 847                 i915_gem_object_unpin_pages(obj);
 848                 remove_mappable_node(&node);
 849         } else {
 850                 i915_gem_object_ggtt_unpin(obj);
 851         }
 852 out:
 853         return ret;
 854 }
 855
 856 static int
 857 i915_gem_shmem_pread(struct drm_device *dev,
 858                      struct drm_i915_gem_object *obj,
 859                      struct drm_i915_gem_pread *args,
 860                      struct drm_file *file)
 861 {
 862         char __user *user_data;
 863         ssize_t remain;
 864         loff_t offset;
 865         int shmem_page_offset, page_length, ret = 0;
 866         int obj_do_bit17_swizzling, page_do_bit17_swizzling;
 867         int prefaulted = 0;
 868         int needs_clflush = 0;
 869         struct sg_page_iter sg_iter;
 870
 871         if (!i915_gem_object_has_struct_page(obj))
 872                 return -ENODEV;
 873
 874         user_data = u64_to_user_ptr(args->data_ptr);
 875         remain = args->size;
 876
 877         obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 878
 879         ret = i915_gem_obj_prepare_shmem_read(obj, &needs_clflush);
 880         if (ret)
 881                 return ret;
 882
 883         offset = args->offset;
 884
 885         for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
 886                          offset >> PAGE_SHIFT) {
 887                 struct page *page = sg_page_iter_page(&sg_iter);
 888
 889                 if (remain <= 0)
 890                         break;
 891
 892                 /* Operation in this page
 893                  *
 894                  * shmem_page_offset = offset within page in shmem file
 895                  * page_length = bytes to copy for this page
 896                  */
 897                 shmem_page_offset = offset_in_page(offset);
 898                 page_length = remain;
 899                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
 900                         page_length = PAGE_SIZE - shmem_page_offset;
 901
 902                 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
 903                         (page_to_phys(page) & (1 << 17)) != 0;
 904
 905                 ret = shmem_pread_fast(page, shmem_page_offset, page_length,
 906                                        user_data, page_do_bit17_swizzling,
 907                                        needs_clflush);
 908                 if (ret == 0)
 909                         goto next_page;
 910
 911                 mutex_unlock(&dev->struct_mutex);
 912
 913                 if (likely(!i915.prefault_disable) && !prefaulted) {
 914                         ret = fault_in_multipages_writeable(user_data, remain);
 915                         /* Userspace is tricking us, but we've already clobbered
 916                          * its pages with the prefault and promised to write the
 917                          * data up to the first fault. Hence ignore any errors
 918                          * and just continue. */
 919                         (void)ret;
 920                         prefaulted = 1;
 921                 }
 922
 923                 ret = shmem_pread_slow(page, shmem_page_offset, page_length,
 924                                        user_data, page_do_bit17_swizzling,
 925                                        needs_clflush);
 926
 927                 mutex_lock(&dev->struct_mutex);
 928
 929                 if (ret)
 930                         goto out;
 931
 932 next_page:
 933                 remain -= page_length;
 934                 user_data += page_length;
 935                 offset += page_length;
 936         }
 937
 938 out:
 939         i915_gem_object_unpin_pages(obj);
 940
 941         return ret;
 942 }
 943
 944 /**
 945  * Reads data from the object referenced by handle.
 946  * @dev: drm device pointer
 947  * @data: ioctl data blob
 948  * @file: drm file pointer
 949  *
 950  * On error, the contents of *data are undefined.
 951  */
 952 int
 953 i915_gem_pread_ioctl(struct drm_device *dev, void *data,
 954                      struct drm_file *file)
 955 {
 956         struct drm_i915_gem_pread *args = data;
 957         struct drm_i915_gem_object *obj;
 958         int ret = 0;
 959
 960         if (args->size == 0)
 961                 return 0;
 962
 963         if (!access_ok(VERIFY_WRITE,
 964                        u64_to_user_ptr(args->data_ptr),
 965                        args->size))
 966                 return -EFAULT;
 967
 968         obj = i915_gem_object_lookup(file, args->handle);
 969         if (!obj)
 970                 return -ENOENT;
 971
 972         /* Bounds check source.  */
 973         if (args->offset > obj->base.size ||
 974             args->size > obj->base.size - args->offset) {
 975                 ret = -EINVAL;
 976                 goto err;
 977         }
 978
 979         trace_i915_gem_object_pread(obj, args->offset, args->size);
 980
 981         ret = __unsafe_wait_rendering(obj, to_rps_client(file), true);
 982         if (ret)
 983                 goto err;
 984
 985         ret = i915_mutex_lock_interruptible(dev);
 986         if (ret)
 987                 goto err;
 988
 989         ret = i915_gem_shmem_pread(dev, obj, args, file);
 990
 991         /* pread for non shmem backed objects */
 992         if (ret == -EFAULT || ret == -ENODEV) {
 993                 intel_runtime_pm_get(to_i915(dev));
 994                 ret = i915_gem_gtt_pread(dev, obj, args->size,
 995                                         args->offset, args->data_ptr);
 996                 intel_runtime_pm_put(to_i915(dev));
 997         }
 998
 999         i915_gem_object_put(obj);
1000         mutex_unlock(&dev->struct_mutex);
1001
1002         return ret;
1003
1004 err:
1005         i915_gem_object_put_unlocked(obj);
1006         return ret;
1007 }
1008
1009 /* This is the fast write path which cannot handle
1010  * page faults in the source data
1011  */
1012
1013 static inline int
1014 fast_user_write(struct io_mapping *mapping,
1015                 loff_t page_base, int page_offset,
1016                 char __user *user_data,
1017                 int length)
1018 {
1019         void __iomem *vaddr_atomic;
1020         void *vaddr;
1021         unsigned long unwritten;
1022
1023         vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
1024         /* We can use the cpu mem copy function because this is X86. */
1025         vaddr = (void __force*)vaddr_atomic + page_offset;
1026         unwritten = __copy_from_user_inatomic_nocache(vaddr,
1027                                                       user_data, length);
1028         io_mapping_unmap_atomic(vaddr_atomic);
1029         return unwritten;
1030 }
1031
1032 /**
1033  * This is the fast pwrite path, where we copy the data directly from the
1034  * user into the GTT, uncached.
1035  * @i915: i915 device private data
1036  * @obj: i915 gem object
1037  * @args: pwrite arguments structure
1038  * @file: drm file pointer
1039  */
1040 static int
1041 i915_gem_gtt_pwrite_fast(struct drm_i915_private *i915,
1042                          struct drm_i915_gem_object *obj,
1043                          struct drm_i915_gem_pwrite *args,
1044                          struct drm_file *file)
1045 {
1046         struct i915_ggtt *ggtt = &i915->ggtt;
1047         struct drm_device *dev = obj->base.dev;
1048         struct drm_mm_node node;
1049         uint64_t remain, offset;
1050         char __user *user_data;
1051         int ret;
1052         bool hit_slow_path = false;
1053
1054         if (i915_gem_object_is_tiled(obj))
1055                 return -EFAULT;
1056
1057         ret = i915_gem_object_ggtt_pin(obj, NULL, 0, 0,
1058                                        PIN_MAPPABLE | PIN_NONBLOCK);
1059         if (ret) {
1060                 ret = insert_mappable_node(i915, &node, PAGE_SIZE);
1061                 if (ret)
1062                         goto out;
1063
1064                 ret = i915_gem_object_get_pages(obj);
1065                 if (ret) {
1066                         remove_mappable_node(&node);
1067                         goto out;
1068                 }
1069
1070                 i915_gem_object_pin_pages(obj);
1071         } else {
1072                 node.start = i915_gem_obj_ggtt_offset(obj);
1073                 node.allocated = false;
1074                 ret = i915_gem_object_put_fence(obj);
1075                 if (ret)
1076                         goto out_unpin;
1077         }
1078
1079         ret = i915_gem_object_set_to_gtt_domain(obj, true);
1080         if (ret)
1081                 goto out_unpin;
1082
1083         intel_fb_obj_invalidate(obj, ORIGIN_GTT);
1084         obj->dirty = true;
1085
1086         user_data = u64_to_user_ptr(args->data_ptr);
1087         offset = args->offset;
1088         remain = args->size;
1089         while (remain) {
1090                 /* Operation in this page
1091                  *
1092                  * page_base = page offset within aperture
1093                  * page_offset = offset within page
1094                  * page_length = bytes to copy for this page
1095                  */
1096                 u32 page_base = node.start;
1097                 unsigned page_offset = offset_in_page(offset);
1098                 unsigned page_length = PAGE_SIZE - page_offset;
1099                 page_length = remain < page_length ? remain : page_length;
1100                 if (node.allocated) {
1101                         wmb(); /* flush the write before we modify the GGTT */
1102                         ggtt->base.insert_page(&ggtt->base,
1103                                                i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT),
1104                                                node.start, I915_CACHE_NONE, 0);
1105                         wmb(); /* flush modifications to the GGTT (insert_page) */
1106                 } else {
1107                         page_base += offset & PAGE_MASK;
1108                 }
1109                 /* If we get a fault while copying data, then (presumably) our
1110                  * source page isn't available.  Return the error and we'll
1111                  * retry in the slow path.
1112                  * If the object is non-shmem backed, we retry again with the
1113                  * path that handles page fault.
1114                  */
1115                 if (fast_user_write(ggtt->mappable, page_base,
1116                                     page_offset, user_data, page_length)) {
1117                         hit_slow_path = true;
1118                         mutex_unlock(&dev->struct_mutex);
1119                         if (slow_user_access(ggtt->mappable,
1120                                              page_base,
1121                                              page_offset, user_data,
1122                                              page_length, true)) {
1123                                 ret = -EFAULT;
1124                                 mutex_lock(&dev->struct_mutex);
1125                                 goto out_flush;
1126                         }
1127
1128                         mutex_lock(&dev->struct_mutex);
1129                 }
1130
1131                 remain -= page_length;
1132                 user_data += page_length;
1133                 offset += page_length;
1134         }
1135
1136 out_flush:
1137         if (hit_slow_path) {
1138                 if (ret == 0 &&
1139                     (obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0) {
1140                         /* The user has modified the object whilst we tried
1141                          * reading from it, and we now have no idea what domain
1142                          * the pages should be in. As we have just been touching
1143                          * them directly, flush everything back to the GTT
1144                          * domain.
1145                          */
1146                         ret = i915_gem_object_set_to_gtt_domain(obj, false);
1147                 }
1148         }
1149
1150         intel_fb_obj_flush(obj, false, ORIGIN_GTT);
1151 out_unpin:
1152         if (node.allocated) {
1153                 wmb();
1154                 ggtt->base.clear_range(&ggtt->base,
1155                                        node.start, node.size,
1156                                        true);
1157                 i915_gem_object_unpin_pages(obj);
1158                 remove_mappable_node(&node);
1159         } else {
1160                 i915_gem_object_ggtt_unpin(obj);
1161         }
1162 out:
1163         return ret;
1164 }
1165
1166 /* Per-page copy function for the shmem pwrite fastpath.
1167  * Flushes invalid cachelines before writing to the target if
1168  * needs_clflush_before is set and flushes out any written cachelines after
1169  * writing if needs_clflush is set. */
1170 static int
1171 shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,
1172                   char __user *user_data,
1173                   bool page_do_bit17_swizzling,
1174                   bool needs_clflush_before,
1175                   bool needs_clflush_after)
1176 {
1177         char *vaddr;
1178         int ret;
1179
1180         if (unlikely(page_do_bit17_swizzling))
1181                 return -EINVAL;
1182
1183         vaddr = kmap_atomic(page);
1184         if (needs_clflush_before)
1185                 drm_clflush_virt_range(vaddr + shmem_page_offset,
1186                                        page_length);
1187         ret = __copy_from_user_inatomic(vaddr + shmem_page_offset,
1188                                         user_data, page_length);
1189         if (needs_clflush_after)
1190                 drm_clflush_virt_range(vaddr + shmem_page_offset,
1191                                        page_length);
1192         kunmap_atomic(vaddr);
1193
1194         return ret ? -EFAULT : 0;
1195 }
1196
1197 /* Only difference to the fast-path function is that this can handle bit17
1198  * and uses non-atomic copy and kmap functions. */
1199 static int
1200 shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
1201                   char __user *user_data,
1202                   bool page_do_bit17_swizzling,
1203                   bool needs_clflush_before,
1204                   bool needs_clflush_after)
1205 {
1206         char *vaddr;
1207         int ret;
1208
1209         vaddr = kmap(page);
1210         if (unlikely(needs_clflush_before || page_do_bit17_swizzling))
1211                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
1212                                              page_length,
1213                                              page_do_bit17_swizzling);
1214         if (page_do_bit17_swizzling)
1215                 ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
1216                                                 user_data,
1217                                                 page_length);
1218         else
1219                 ret = __copy_from_user(vaddr + shmem_page_offset,
1220                                        user_data,
1221                                        page_length);
1222         if (needs_clflush_after)
1223                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
1224                                              page_length,
1225                                              page_do_bit17_swizzling);
1226         kunmap(page);
1227
1228         return ret ? -EFAULT : 0;
1229 }
1230
1231 static int
1232 i915_gem_shmem_pwrite(struct drm_device *dev,
1233                       struct drm_i915_gem_object *obj,
1234                       struct drm_i915_gem_pwrite *args,
1235                       struct drm_file *file)
1236 {
1237         ssize_t remain;
1238         loff_t offset;
1239         char __user *user_data;
1240         int shmem_page_offset, page_length, ret = 0;
1241         int obj_do_bit17_swizzling, page_do_bit17_swizzling;
1242         int hit_slowpath = 0;
1243         int needs_clflush_after = 0;
1244         int needs_clflush_before = 0;
1245         struct sg_page_iter sg_iter;
1246
1247         user_data = u64_to_user_ptr(args->data_ptr);
1248         remain = args->size;
1249
1250         obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
1251
1252         ret = i915_gem_object_wait_rendering(obj, false);
1253         if (ret)
1254                 return ret;
1255
1256         if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
1257                 /* If we're not in the cpu write domain, set ourself into the gtt
1258                  * write domain and manually flush cachelines (if required). This
1259                  * optimizes for the case when the gpu will use the data
1260                  * right away and we therefore have to clflush anyway. */
1261                 needs_clflush_after = cpu_write_needs_clflush(obj);
1262         }
1263         /* Same trick applies to invalidate partially written cachelines read
1264          * before writing. */
1265         if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0)
1266                 needs_clflush_before =
1267                         !cpu_cache_is_coherent(dev, obj->cache_level);
1268
1269         ret = i915_gem_object_get_pages(obj);
1270         if (ret)
1271                 return ret;
1272
1273         intel_fb_obj_invalidate(obj, ORIGIN_CPU);
1274
1275         i915_gem_object_pin_pages(obj);
1276
1277         offset = args->offset;
1278         obj->dirty = 1;
1279
1280         for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
1281                          offset >> PAGE_SHIFT) {
1282                 struct page *page = sg_page_iter_page(&sg_iter);
1283                 int partial_cacheline_write;
1284
1285                 if (remain <= 0)
1286                         break;
1287
1288                 /* Operation in this page
1289                  *
1290                  * shmem_page_offset = offset within page in shmem file
1291                  * page_length = bytes to copy for this page
1292                  */
1293                 shmem_page_offset = offset_in_page(offset);
1294
1295                 page_length = remain;
1296                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
1297                         page_length = PAGE_SIZE - shmem_page_offset;
1298
1299                 /* If we don't overwrite a cacheline completely we need to be
1300                  * careful to have up-to-date data by first clflushing. Don't
1301                  * overcomplicate things and flush the entire patch. */
1302                 partial_cacheline_write = needs_clflush_before &&
1303                         ((shmem_page_offset | page_length)
1304                                 & (boot_cpu_data.x86_clflush_size - 1));
1305
1306                 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
1307                         (page_to_phys(page) & (1 << 17)) != 0;
1308
1309                 ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
1310                                         user_data, page_do_bit17_swizzling,
1311                                         partial_cacheline_write,
1312                                         needs_clflush_after);
1313                 if (ret == 0)
1314                         goto next_page;
1315
1316                 hit_slowpath = 1;
1317                 mutex_unlock(&dev->struct_mutex);
1318                 ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
1319                                         user_data, page_do_bit17_swizzling,
1320                                         partial_cacheline_write,
1321                                         needs_clflush_after);
1322
1323                 mutex_lock(&dev->struct_mutex);
1324
1325                 if (ret)
1326                         goto out;
1327
1328 next_page:
1329                 remain -= page_length;
1330                 user_data += page_length;
1331                 offset += page_length;
1332         }
1333
1334 out:
1335         i915_gem_object_unpin_pages(obj);
1336
1337         if (hit_slowpath) {
1338                 /*
1339                  * Fixup: Flush cpu caches in case we didn't flush the dirty
1340                  * cachelines in-line while writing and the object moved
1341                  * out of the cpu write domain while we've dropped the lock.
1342                  */
1343                 if (!needs_clflush_after &&
1344                     obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
1345                         if (i915_gem_clflush_object(obj, obj->pin_display))
1346                                 needs_clflush_after = true;
1347                 }
1348         }
1349
1350         if (needs_clflush_after)
1351                 i915_gem_chipset_flush(to_i915(dev));
1352         else
1353                 obj->cache_dirty = true;
1354
1355         intel_fb_obj_flush(obj, false, ORIGIN_CPU);
1356         return ret;
1357 }
1358
1359 /**
1360  * Writes data to the object referenced by handle.
1361  * @dev: drm device
1362  * @data: ioctl data blob
1363  * @file: drm file
1364  *
1365  * On error, the contents of the buffer that were to be modified are undefined.
1366  */
1367 int
1368 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
1369                       struct drm_file *file)
1370 {
1371         struct drm_i915_private *dev_priv = to_i915(dev);
1372         struct drm_i915_gem_pwrite *args = data;
1373         struct drm_i915_gem_object *obj;
1374         int ret;
1375
1376         if (args->size == 0)
1377                 return 0;
1378
1379         if (!access_ok(VERIFY_READ,
1380                        u64_to_user_ptr(args->data_ptr),
1381                        args->size))
1382                 return -EFAULT;
1383
1384         if (likely(!i915.prefault_disable)) {
1385                 ret = fault_in_multipages_readable(u64_to_user_ptr(args->data_ptr),
1386                                                    args->size);
1387                 if (ret)
1388                         return -EFAULT;
1389         }
1390
1391         obj = i915_gem_object_lookup(file, args->handle);
1392         if (!obj)
1393                 return -ENOENT;
1394
1395         /* Bounds check destination. */
1396         if (args->offset > obj->base.size ||
1397             args->size > obj->base.size - args->offset) {
1398                 ret = -EINVAL;
1399                 goto err;
1400         }
1401
1402         trace_i915_gem_object_pwrite(obj, args->offset, args->size);
1403
1404         ret = __unsafe_wait_rendering(obj, to_rps_client(file), false);
1405         if (ret)
1406                 goto err;
1407
1408         intel_runtime_pm_get(dev_priv);
1409
1410         ret = i915_mutex_lock_interruptible(dev);
1411         if (ret)
1412                 goto err_rpm;
1413
1414         ret = -EFAULT;
1415         /* We can only do the GTT pwrite on untiled buffers, as otherwise
1416          * it would end up going through the fenced access, and we'll get
1417          * different detiling behavior between reading and writing.
1418          * pread/pwrite currently are reading and writing from the CPU
1419          * perspective, requiring manual detiling by the client.
1420          */
1421         if (!i915_gem_object_has_struct_page(obj) ||
1422             cpu_write_needs_clflush(obj)) {
1423                 ret = i915_gem_gtt_pwrite_fast(dev_priv, obj, args, file);
1424                 /* Note that the gtt paths might fail with non-page-backed user
1425                  * pointers (e.g. gtt mappings when moving data between
1426                  * textures). Fallback to the shmem path in that case. */
1427         }
1428
1429         if (ret == -EFAULT || ret == -ENOSPC) {
1430                 if (obj->phys_handle)
1431                         ret = i915_gem_phys_pwrite(obj, args, file);
1432                 else if (i915_gem_object_has_struct_page(obj))
1433                         ret = i915_gem_shmem_pwrite(dev, obj, args, file);
1434                 else
1435                         ret = -ENODEV;
1436         }
1437
1438         i915_gem_object_put(obj);
1439         mutex_unlock(&dev->struct_mutex);
1440         intel_runtime_pm_put(dev_priv);
1441
1442         return ret;
1443
1444 err_rpm:
1445         intel_runtime_pm_put(dev_priv);
1446 err:
1447         i915_gem_object_put_unlocked(obj);
1448         return ret;
1449 }
1450
1451 static enum fb_op_origin
1452 write_origin(struct drm_i915_gem_object *obj, unsigned domain)
1453 {
1454         return domain == I915_GEM_DOMAIN_GTT && !obj->has_wc_mmap ?
1455                ORIGIN_GTT : ORIGIN_CPU;
1456 }
1457
1458 /**
1459  * Called when user space prepares to use an object with the CPU, either
1460  * through the mmap ioctl's mapping or a GTT mapping.
1461  * @dev: drm device
1462  * @data: ioctl data blob
1463  * @file: drm file
1464  */
1465 int
1466 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1467                           struct drm_file *file)
1468 {
1469         struct drm_i915_gem_set_domain *args = data;
1470         struct drm_i915_gem_object *obj;
1471         uint32_t read_domains = args->read_domains;
1472         uint32_t write_domain = args->write_domain;
1473         int ret;
1474
1475         /* Only handle setting domains to types used by the CPU. */
1476         if ((write_domain | read_domains) & I915_GEM_GPU_DOMAINS)
1477                 return -EINVAL;
1478
1479         /* Having something in the write domain implies it's in the read
1480          * domain, and only that read domain.  Enforce that in the request.
1481          */
1482         if (write_domain != 0 && read_domains != write_domain)
1483                 return -EINVAL;
1484
1485         obj = i915_gem_object_lookup(file, args->handle);
1486         if (!obj)
1487                 return -ENOENT;
1488
1489         /* Try to flush the object off the GPU without holding the lock.
1490          * We will repeat the flush holding the lock in the normal manner
1491          * to catch cases where we are gazumped.
1492          */
1493         ret = __unsafe_wait_rendering(obj, to_rps_client(file), !write_domain);
1494         if (ret)
1495                 goto err;
1496
1497         ret = i915_mutex_lock_interruptible(dev);
1498         if (ret)
1499                 goto err;
1500
1501         if (read_domains & I915_GEM_DOMAIN_GTT)
1502                 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
1503         else
1504                 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
1505
1506         if (write_domain != 0)
1507                 intel_fb_obj_invalidate(obj, write_origin(obj, write_domain));
1508
1509         i915_gem_object_put(obj);
1510         mutex_unlock(&dev->struct_mutex);
1511         return ret;
1512
1513 err:
1514         i915_gem_object_put_unlocked(obj);
1515         return ret;
1516 }
1517
1518 /**
1519  * Called when user space has done writes to this buffer
1520  * @dev: drm device
1521  * @data: ioctl data blob
1522  * @file: drm file
1523  */
1524 int
1525 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1526                          struct drm_file *file)
1527 {
1528         struct drm_i915_gem_sw_finish *args = data;
1529         struct drm_i915_gem_object *obj;
1530         int err = 0;
1531
1532         obj = i915_gem_object_lookup(file, args->handle);
1533         if (!obj)
1534                 return -ENOENT;
1535
1536         /* Pinned buffers may be scanout, so flush the cache */
1537         if (READ_ONCE(obj->pin_display)) {
1538                 err = i915_mutex_lock_interruptible(dev);
1539                 if (!err) {
1540                         i915_gem_object_flush_cpu_write_domain(obj);
1541                         mutex_unlock(&dev->struct_mutex);
1542                 }
1543         }
1544
1545         i915_gem_object_put_unlocked(obj);
1546         return err;
1547 }
1548
1549 /**
1550  * i915_gem_mmap_ioctl - Maps the contents of an object, returning the address
1551  *                       it is mapped to.
1552  * @dev: drm device
1553  * @data: ioctl data blob
1554  * @file: drm file
1555  *
1556  * While the mapping holds a reference on the contents of the object, it doesn't
1557  * imply a ref on the object itself.
1558  *
1559  * IMPORTANT:
1560  *
1561  * DRM driver writers who look a this function as an example for how to do GEM
1562  * mmap support, please don't implement mmap support like here. The modern way
1563  * to implement DRM mmap support is with an mmap offset ioctl (like
1564  * i915_gem_mmap_gtt) and then using the mmap syscall on the DRM fd directly.
1565  * That way debug tooling like valgrind will understand what's going on, hiding
1566  * the mmap call in a driver private ioctl will break that. The i915 driver only
1567  * does cpu mmaps this way because we didn't know better.
1568  */
1569 int
1570 i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1571                     struct drm_file *file)
1572 {
1573         struct drm_i915_gem_mmap *args = data;
1574         struct drm_i915_gem_object *obj;
1575         unsigned long addr;
1576
1577         if (args->flags & ~(I915_MMAP_WC))
1578                 return -EINVAL;
1579
1580         if (args->flags & I915_MMAP_WC && !boot_cpu_has(X86_FEATURE_PAT))
1581                 return -ENODEV;
1582
1583         obj = i915_gem_object_lookup(file, args->handle);
1584         if (!obj)
1585                 return -ENOENT;
1586
1587         /* prime objects have no backing filp to GEM mmap
1588          * pages from.
1589          */
1590         if (!obj->base.filp) {
1591                 i915_gem_object_put_unlocked(obj);
1592                 return -EINVAL;
1593         }
1594
1595         addr = vm_mmap(obj->base.filp, 0, args->size,
1596                        PROT_READ | PROT_WRITE, MAP_SHARED,
1597                        args->offset);
1598         if (args->flags & I915_MMAP_WC) {
1599                 struct mm_struct *mm = current->mm;
1600                 struct vm_area_struct *vma;
1601
1602                 if (down_write_killable(&mm->mmap_sem)) {
1603                         i915_gem_object_put_unlocked(obj);
1604                         return -EINTR;
1605                 }
1606                 vma = find_vma(mm, addr);
1607                 if (vma)
1608                         vma->vm_page_prot =
1609                                 pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
1610                 else
1611                         addr = -ENOMEM;
1612                 up_write(&mm->mmap_sem);
1613
1614                 /* This may race, but that's ok, it only gets set */
1615                 WRITE_ONCE(obj->has_wc_mmap, true);
1616         }
1617         i915_gem_object_put_unlocked(obj);
1618         if (IS_ERR((void *)addr))
1619                 return addr;
1620
1621         args->addr_ptr = (uint64_t) addr;
1622
1623         return 0;
1624 }
1625
1626 /**
1627  * i915_gem_fault - fault a page into the GTT
1628  * @vma: VMA in question
1629  * @vmf: fault info
1630  *
1631  * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1632  * from userspace.  The fault handler takes care of binding the object to
1633  * the GTT (if needed), allocating and programming a fence register (again,
1634  * only if needed based on whether the old reg is still valid or the object
1635  * is tiled) and inserting a new PTE into the faulting process.
1636  *
1637  * Note that the faulting process may involve evicting existing objects
1638  * from the GTT and/or fence registers to make room.  So performance may
1639  * suffer if the GTT working set is large or there are few fence registers
1640  * left.
1641  */
1642 int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1643 {
1644         struct drm_i915_gem_object *obj = to_intel_bo(vma->vm_private_data);
1645         struct drm_device *dev = obj->base.dev;
1646         struct drm_i915_private *dev_priv = to_i915(dev);
1647         struct i915_ggtt *ggtt = &dev_priv->ggtt;
1648         struct i915_ggtt_view view = i915_ggtt_view_normal;
1649         bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
1650         pgoff_t page_offset;
1651         unsigned long pfn;
1652         int ret;
1653
1654         /* We don't use vmf->pgoff since that has the fake offset */
1655         page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
1656                 PAGE_SHIFT;
1657
1658         trace_i915_gem_object_fault(obj, page_offset, true, write);
1659
1660         /* Try to flush the object off the GPU first without holding the lock.
1661          * Upon acquiring the lock, we will perform our sanity checks and then
1662          * repeat the flush holding the lock in the normal manner to catch cases
1663          * where we are gazumped.
1664          */
1665         ret = __unsafe_wait_rendering(obj, NULL, !write);
1666         if (ret)
1667                 goto err;
1668
1669         intel_runtime_pm_get(dev_priv);
1670
1671         ret = i915_mutex_lock_interruptible(dev);
1672         if (ret)
1673                 goto err_rpm;
1674
1675         /* Access to snoopable pages through the GTT is incoherent. */
1676         if (obj->cache_level != I915_CACHE_NONE && !HAS_LLC(dev)) {
1677                 ret = -EFAULT;
1678                 goto err_unlock;
1679         }
1680
1681         /* Use a partial view if the object is bigger than the aperture. */
1682         if (obj->base.size >= ggtt->mappable_end &&
1683             !i915_gem_object_is_tiled(obj)) {
1684                 static const unsigned int chunk_size = 256; // 1 MiB
1685
1686                 memset(&view, 0, sizeof(view));
1687                 view.type = I915_GGTT_VIEW_PARTIAL;
1688                 view.params.partial.offset = rounddown(page_offset, chunk_size);
1689                 view.params.partial.size =
1690                         min_t(unsigned int,
1691                               chunk_size,
1692                               (vma->vm_end - vma->vm_start)/PAGE_SIZE -
1693                               view.params.partial.offset);
1694         }
1695
1696         /* Now pin it into the GTT if needed */
1697         ret = i915_gem_object_ggtt_pin(obj, &view, 0, 0, PIN_MAPPABLE);
1698         if (ret)
1699                 goto err_unlock;
1700
1701         ret = i915_gem_object_set_to_gtt_domain(obj, write);
1702         if (ret)
1703                 goto err_unpin;
1704
1705         ret = i915_gem_object_get_fence(obj);
1706         if (ret)
1707                 goto err_unpin;
1708
1709         /* Finally, remap it using the new GTT offset */
1710         pfn = ggtt->mappable_base +
1711                 i915_gem_obj_ggtt_offset_view(obj, &view);
1712         pfn >>= PAGE_SHIFT;
1713
1714         if (unlikely(view.type == I915_GGTT_VIEW_PARTIAL)) {
1715                 /* Overriding existing pages in partial view does not cause
1716                  * us any trouble as TLBs are still valid because the fault
1717                  * is due to userspace losing part of the mapping or never
1718                  * having accessed it before (at this partials' range).
1719                  */
1720                 unsigned long base = vma->vm_start +
1721                                      (view.params.partial.offset << PAGE_SHIFT);
1722                 unsigned int i;
1723
1724                 for (i = 0; i < view.params.partial.size; i++) {
1725                         ret = vm_insert_pfn(vma, base + i * PAGE_SIZE, pfn + i);
1726                         if (ret)
1727                                 break;
1728                 }
1729
1730                 obj->fault_mappable = true;
1731         } else {
1732                 if (!obj->fault_mappable) {
1733                         unsigned long size = min_t(unsigned long,
1734                                                    vma->vm_end - vma->vm_start,
1735                                                    obj->base.size);
1736                         int i;
1737
1738                         for (i = 0; i < size >> PAGE_SHIFT; i++) {
1739                                 ret = vm_insert_pfn(vma,
1740                                                     (unsigned long)vma->vm_start + i * PAGE_SIZE,
1741                                                     pfn + i);
1742                                 if (ret)
1743                                         break;
1744                         }
1745
1746                         obj->fault_mappable = true;
1747                 } else
1748                         ret = vm_insert_pfn(vma,
1749                                             (unsigned long)vmf->virtual_address,
1750                                             pfn + page_offset);
1751         }
1752 err_unpin:
1753         i915_gem_object_ggtt_unpin_view(obj, &view);
1754 err_unlock:
1755         mutex_unlock(&dev->struct_mutex);
1756 err_rpm:
1757         intel_runtime_pm_put(dev_priv);
1758 err:
1759         switch (ret) {
1760         case -EIO:
1761                 /*
1762                  * We eat errors when the gpu is terminally wedged to avoid
1763                  * userspace unduly crashing (gl has no provisions for mmaps to
1764                  * fail). But any other -EIO isn't ours (e.g. swap in failure)
1765                  * and so needs to be reported.
1766                  */
1767                 if (!i915_terminally_wedged(&dev_priv->gpu_error)) {
1768                         ret = VM_FAULT_SIGBUS;
1769                         break;
1770                 }
1771         case -EAGAIN:
1772                 /*
1773                  * EAGAIN means the gpu is hung and we'll wait for the error
1774                  * handler to reset everything when re-faulting in
1775                  * i915_mutex_lock_interruptible.
1776                  */
1777         case 0:
1778         case -ERESTARTSYS:
1779         case -EINTR:
1780         case -EBUSY:
1781                 /*
1782                  * EBUSY is ok: this just means that another thread
1783                  * already did the job.
1784                  */
1785                 ret = VM_FAULT_NOPAGE;
1786                 break;
1787         case -ENOMEM:
1788                 ret = VM_FAULT_OOM;
1789                 break;
1790         case -ENOSPC:
1791         case -EFAULT:
1792                 ret = VM_FAULT_SIGBUS;
1793                 break;
1794         default:
1795                 WARN_ONCE(ret, "unhandled error in i915_gem_fault: %i\n", ret);
1796                 ret = VM_FAULT_SIGBUS;
1797                 break;
1798         }
1799         return ret;
1800 }
1801
1802 /**
1803  * i915_gem_release_mmap - remove physical page mappings
1804  * @obj: obj in question
1805  *
1806  * Preserve the reservation of the mmapping with the DRM core code, but
1807  * relinquish ownership of the pages back to the system.
1808  *
1809  * It is vital that we remove the page mapping if we have mapped a tiled
1810  * object through the GTT and then lose the fence register due to
1811  * resource pressure. Similarly if the object has been moved out of the
1812  * aperture, than pages mapped into userspace must be revoked. Removing the
1813  * mapping will then trigger a page fault on the next user access, allowing
1814  * fixup by i915_gem_fault().
1815  */
1816 void
1817 i915_gem_release_mmap(struct drm_i915_gem_object *obj)
1818 {
1819         /* Serialisation between user GTT access and our code depends upon
1820          * revoking the CPU's PTE whilst the mutex is held. The next user
1821          * pagefault then has to wait until we release the mutex.
1822          */
1823         lockdep_assert_held(&obj->base.dev->struct_mutex);
1824
1825         if (!obj->fault_mappable)
1826                 return;
1827
1828         drm_vma_node_unmap(&obj->base.vma_node,
1829                            obj->base.dev->anon_inode->i_mapping);
1830
1831         /* Ensure that the CPU's PTE are revoked and there are not outstanding
1832          * memory transactions from userspace before we return. The TLB
1833          * flushing implied above by changing the PTE above *should* be
1834          * sufficient, an extra barrier here just provides us with a bit
1835          * of paranoid documentation about our requirement to serialise
1836          * memory writes before touching registers / GSM.
1837          */
1838         wmb();
1839
1840         obj->fault_mappable = false;
1841 }
1842
1843 void
1844 i915_gem_release_all_mmaps(struct drm_i915_private *dev_priv)
1845 {
1846         struct drm_i915_gem_object *obj;
1847
1848         list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list)
1849                 i915_gem_release_mmap(obj);
1850 }
1851
1852 /**
1853  * i915_gem_get_ggtt_size - return required global GTT size for an object
1854  * @dev_priv: i915 device
1855  * @size: object size
1856  * @tiling_mode: tiling mode
1857  *
1858  * Return the required global GTT size for an object, taking into account
1859  * potential fence register mapping.
1860  */
1861 u64 i915_gem_get_ggtt_size(struct drm_i915_private *dev_priv,
1862                            u64 size, int tiling_mode)
1863 {
1864         u64 ggtt_size;
1865
1866         GEM_BUG_ON(size == 0);
1867
1868         if (INTEL_GEN(dev_priv) >= 4 ||
1869             tiling_mode == I915_TILING_NONE)
1870                 return size;
1871
1872         /* Previous chips need a power-of-two fence region when tiling */
1873         if (IS_GEN3(dev_priv))
1874                 ggtt_size = 1024*1024;
1875         else
1876                 ggtt_size = 512*1024;
1877
1878         while (ggtt_size < size)
1879                 ggtt_size <<= 1;
1880
1881         return ggtt_size;
1882 }
1883
1884 /**
1885  * i915_gem_get_ggtt_alignment - return required global GTT alignment
1886  * @dev_priv: i915 device
1887  * @size: object size
1888  * @tiling_mode: tiling mode
1889  * @fenced: is fenced alignment required or not
1890  *
1891  * Return the required global GTT alignment for an object, taking into account
1892  * potential fence register mapping.
1893  */
1894 u64 i915_gem_get_ggtt_alignment(struct drm_i915_private *dev_priv, u64 size,
1895                                 int tiling_mode, bool fenced)
1896 {
1897         GEM_BUG_ON(size == 0);
1898
1899         /*
1900          * Minimum alignment is 4k (GTT page size), but might be greater
1901          * if a fence register is needed for the object.
1902          */
1903         if (INTEL_GEN(dev_priv) >= 4 || (!fenced && IS_G33(dev_priv)) ||
1904             tiling_mode == I915_TILING_NONE)
1905                 return 4096;
1906
1907         /*
1908          * Previous chips need to be aligned to the size of the smallest
1909          * fence register that can contain the object.
1910          */
1911         return i915_gem_get_ggtt_size(dev_priv, size, tiling_mode);
1912 }
1913
1914 static int i915_gem_object_create_mmap_offset(struct drm_i915_gem_object *obj)
1915 {
1916         struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
1917         int err;
1918
1919         err = drm_gem_create_mmap_offset(&obj->base);
1920         if (!err)
1921                 return 0;
1922
1923         /* We can idle the GPU locklessly to flush stale objects, but in order
1924          * to claim that space for ourselves, we need to take the big
1925          * struct_mutex to free the requests+objects and allocate our slot.
1926          */
1927         err = i915_gem_wait_for_idle(dev_priv, true);
1928         if (err)
1929                 return err;
1930
1931         err = i915_mutex_lock_interruptible(&dev_priv->drm);
1932         if (!err) {
1933                 i915_gem_retire_requests(dev_priv);
1934                 err = drm_gem_create_mmap_offset(&obj->base);
1935                 mutex_unlock(&dev_priv->drm.struct_mutex);
1936         }
1937
1938         return err;
1939 }
1940
1941 static void i915_gem_object_free_mmap_offset(struct drm_i915_gem_object *obj)
1942 {
1943         drm_gem_free_mmap_offset(&obj->base);
1944 }
1945
1946 int
1947 i915_gem_mmap_gtt(struct drm_file *file,
1948                   struct drm_device *dev,
1949                   uint32_t handle,
1950                   uint64_t *offset)
1951 {
1952         struct drm_i915_gem_object *obj;
1953         int ret;
1954
1955         obj = i915_gem_object_lookup(file, handle);
1956         if (!obj)
1957                 return -ENOENT;
1958
1959         ret = i915_gem_object_create_mmap_offset(obj);
1960         if (ret == 0)
1961                 *offset = drm_vma_node_offset_addr(&obj->base.vma_node);
1962
1963         i915_gem_object_put_unlocked(obj);
1964         return ret;
1965 }
1966
1967 /**
1968  * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
1969  * @dev: DRM device
1970  * @data: GTT mapping ioctl data
1971  * @file: GEM object info
1972  *
1973  * Simply returns the fake offset to userspace so it can mmap it.
1974  * The mmap call will end up in drm_gem_mmap(), which will set things
1975  * up so we can get faults in the handler above.
1976  *
1977  * The fault handler will take care of binding the object into the GTT
1978  * (since it may have been evicted to make room for something), allocating
1979  * a fence register, and mapping the appropriate aperture address into
1980  * userspace.
1981  */
1982 int
1983 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
1984                         struct drm_file *file)
1985 {
1986         struct drm_i915_gem_mmap_gtt *args = data;
1987
1988         return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);
1989 }
1990
1991 /* Immediately discard the backing storage */
1992 static void
1993 i915_gem_object_truncate(struct drm_i915_gem_object *obj)
1994 {
1995         i915_gem_object_free_mmap_offset(obj);
1996
1997         if (obj->base.filp == NULL)
1998                 return;
1999
2000         /* Our goal here is to return as much of the memory as
2001          * is possible back to the system as we are called from OOM.
2002          * To do this we must instruct the shmfs to drop all of its
2003          * backing pages, *now*.
2004          */
2005         shmem_truncate_range(file_inode(obj->base.filp), 0, (loff_t)-1);
2006         obj->madv = __I915_MADV_PURGED;
2007 }
2008
2009 /* Try to discard unwanted pages */
2010 static void
2011 i915_gem_object_invalidate(struct drm_i915_gem_object *obj)
2012 {
2013         struct address_space *mapping;
2014
2015         switch (obj->madv) {
2016         case I915_MADV_DONTNEED:
2017                 i915_gem_object_truncate(obj);
2018         case __I915_MADV_PURGED:
2019                 return;
2020         }
2021
2022         if (obj->base.filp == NULL)
2023                 return;
2024
2025         mapping = file_inode(obj->base.filp)->i_mapping,
2026         invalidate_mapping_pages(mapping, 0, (loff_t)-1);
2027 }
2028
2029 static void
2030 i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
2031 {
2032         struct sgt_iter sgt_iter;
2033         struct page *page;
2034         int ret;
2035
2036         BUG_ON(obj->madv == __I915_MADV_PURGED);
2037
2038         ret = i915_gem_object_set_to_cpu_domain(obj, true);
2039         if (WARN_ON(ret)) {
2040                 /* In the event of a disaster, abandon all caches and
2041                  * hope for the best.
2042                  */
2043                 i915_gem_clflush_object(obj, true);
2044                 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2045         }
2046
2047         i915_gem_gtt_finish_object(obj);
2048
2049         if (i915_gem_object_needs_bit17_swizzle(obj))
2050                 i915_gem_object_save_bit_17_swizzle(obj);
2051
2052         if (obj->madv == I915_MADV_DONTNEED)
2053                 obj->dirty = 0;
2054
2055         for_each_sgt_page(page, sgt_iter, obj->pages) {
2056                 if (obj->dirty)
2057                         set_page_dirty(page);
2058
2059                 if (obj->madv == I915_MADV_WILLNEED)
2060                         mark_page_accessed(page);
2061
2062                 put_page(page);
2063         }
2064         obj->dirty = 0;
2065
2066         sg_free_table(obj->pages);
2067         kfree(obj->pages);
2068 }
2069
2070 int
2071 i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
2072 {
2073         const struct drm_i915_gem_object_ops *ops = obj->ops;
2074
2075         if (obj->pages == NULL)
2076                 return 0;
2077
2078         if (obj->pages_pin_count)
2079                 return -EBUSY;
2080
2081         GEM_BUG_ON(obj->bind_count);
2082
2083         /* ->put_pages might need to allocate memory for the bit17 swizzle
2084          * array, hence protect them from being reaped by removing them from gtt
2085          * lists early. */
2086         list_del(&obj->global_list);
2087
2088         if (obj->mapping) {
2089                 /* low bits are ignored by is_vmalloc_addr and kmap_to_page */
2090                 if (is_vmalloc_addr(obj->mapping))
2091                         vunmap(obj->mapping);
2092                 else
2093                         kunmap(kmap_to_page(obj->mapping));
2094                 obj->mapping = NULL;
2095         }
2096
2097         ops->put_pages(obj);
2098         obj->pages = NULL;
2099
2100         i915_gem_object_invalidate(obj);
2101
2102         return 0;
2103 }
2104
2105 static int
2106 i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
2107 {
2108         struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
2109         int page_count, i;
2110         struct address_space *mapping;
2111         struct sg_table *st;
2112         struct scatterlist *sg;
2113         struct sgt_iter sgt_iter;
2114         struct page *page;
2115         unsigned long last_pfn = 0;     /* suppress gcc warning */
2116         int ret;
2117         gfp_t gfp;
2118
2119         /* Assert that the object is not currently in any GPU domain. As it
2120          * wasn't in the GTT, there shouldn't be any way it could have been in
2121          * a GPU cache
2122          */
2123         BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
2124         BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
2125
2126         st = kmalloc(sizeof(*st), GFP_KERNEL);
2127         if (st == NULL)
2128                 return -ENOMEM;
2129
2130         page_count = obj->base.size / PAGE_SIZE;
2131         if (sg_alloc_table(st, page_count, GFP_KERNEL)) {
2132                 kfree(st);
2133                 return -ENOMEM;
2134         }
2135
2136         /* Get the list of pages out of our struct file.  They'll be pinned
2137          * at this point until we release them.
2138          *
2139          * Fail silently without starting the shrinker
2140          */
2141         mapping = file_inode(obj->base.filp)->i_mapping;
2142         gfp = mapping_gfp_constraint(mapping, ~(__GFP_IO | __GFP_RECLAIM));
2143         gfp |= __GFP_NORETRY | __GFP_NOWARN;
2144         sg = st->sgl;
2145         st->nents = 0;
2146         for (i = 0; i < page_count; i++) {
2147                 page = shmem_read_mapping_page_gfp(mapping, i, gfp);
2148                 if (IS_ERR(page)) {
2149                         i915_gem_shrink(dev_priv,
2150                                         page_count,
2151                                         I915_SHRINK_BOUND |
2152                                         I915_SHRINK_UNBOUND |
2153                                         I915_SHRINK_PURGEABLE);
2154                         page = shmem_read_mapping_page_gfp(mapping, i, gfp);
2155                 }
2156                 if (IS_ERR(page)) {
2157                         /* We've tried hard to allocate the memory by reaping
2158                          * our own buffer, now let the real VM do its job and
2159                          * go down in flames if truly OOM.
2160                          */
2161                         i915_gem_shrink_all(dev_priv);
2162                         page = shmem_read_mapping_page(mapping, i);
2163                         if (IS_ERR(page)) {
2164                                 ret = PTR_ERR(page);
2165                                 goto err_pages;
2166                         }
2167                 }
2168 #ifdef CONFIG_SWIOTLB
2169                 if (swiotlb_nr_tbl()) {
2170                         st->nents++;
2171                         sg_set_page(sg, page, PAGE_SIZE, 0);
2172                         sg = sg_next(sg);
2173                         continue;
2174                 }
2175 #endif
2176                 if (!i || page_to_pfn(page) != last_pfn + 1) {
2177                         if (i)
2178                                 sg = sg_next(sg);
2179                         st->nents++;
2180                         sg_set_page(sg, page, PAGE_SIZE, 0);
2181                 } else {
2182                         sg->length += PAGE_SIZE;
2183                 }
2184                 last_pfn = page_to_pfn(page);
2185
2186                 /* Check that the i965g/gm workaround works. */
2187                 WARN_ON((gfp & __GFP_DMA32) && (last_pfn >= 0x00100000UL));
2188         }
2189 #ifdef CONFIG_SWIOTLB
2190         if (!swiotlb_nr_tbl())
2191 #endif
2192                 sg_mark_end(sg);
2193         obj->pages = st;
2194
2195         ret = i915_gem_gtt_prepare_object(obj);
2196         if (ret)
2197                 goto err_pages;
2198
2199         if (i915_gem_object_needs_bit17_swizzle(obj))
2200                 i915_gem_object_do_bit_17_swizzle(obj);
2201
2202         if (i915_gem_object_is_tiled(obj) &&
2203             dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES)
2204                 i915_gem_object_pin_pages(obj);
2205
2206         return 0;
2207
2208 err_pages:
2209         sg_mark_end(sg);
2210         for_each_sgt_page(page, sgt_iter, st)
2211                 put_page(page);
2212         sg_free_table(st);
2213         kfree(st);
2214
2215         /* shmemfs first checks if there is enough memory to allocate the page
2216          * and reports ENOSPC should there be insufficient, along with the usual
2217          * ENOMEM for a genuine allocation failure.
2218          *
2219          * We use ENOSPC in our driver to mean that we have run out of aperture
2220          * space and so want to translate the error from shmemfs back to our
2221          * usual understanding of ENOMEM.
2222          */
2223         if (ret == -ENOSPC)
2224                 ret = -ENOMEM;
2225
2226         return ret;
2227 }
2228
2229 /* Ensure that the associated pages are gathered from the backing storage
2230  * and pinned into our object. i915_gem_object_get_pages() may be called
2231  * multiple times before they are released by a single call to
2232  * i915_gem_object_put_pages() - once the pages are no longer referenced
2233  * either as a result of memory pressure (reaping pages under the shrinker)
2234  * or as the object is itself released.
2235  */
2236 int
2237 i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
2238 {
2239         struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
2240         const struct drm_i915_gem_object_ops *ops = obj->ops;
2241         int ret;
2242
2243         if (obj->pages)
2244                 return 0;
2245
2246         if (obj->madv != I915_MADV_WILLNEED) {
2247                 DRM_DEBUG("Attempting to obtain a purgeable object\n");
2248                 return -EFAULT;
2249         }
2250
2251         BUG_ON(obj->pages_pin_count);
2252
2253         ret = ops->get_pages(obj);
2254         if (ret)
2255                 return ret;
2256
2257         list_add_tail(&obj->global_list, &dev_priv->mm.unbound_list);
2258
2259         obj->get_page.sg = obj->pages->sgl;
2260         obj->get_page.last = 0;
2261
2262         return 0;
2263 }
2264
2265 /* The 'mapping' part of i915_gem_object_pin_map() below */
2266 static void *i915_gem_object_map(const struct drm_i915_gem_object *obj,
2267                                  enum i915_map_type type)
2268 {
2269         unsigned long n_pages = obj->base.size >> PAGE_SHIFT;
2270         struct sg_table *sgt = obj->pages;
2271         struct sgt_iter sgt_iter;
2272         struct page *page;
2273         struct page *stack_pages[32];
2274         struct page **pages = stack_pages;
2275         unsigned long i = 0;
2276         pgprot_t pgprot;
2277         void *addr;
2278
2279         /* A single page can always be kmapped */
2280         if (n_pages == 1 && type == I915_MAP_WB)
2281                 return kmap(sg_page(sgt->sgl));
2282
2283         if (n_pages > ARRAY_SIZE(stack_pages)) {
2284                 /* Too big for stack -- allocate temporary array instead */
2285                 pages = drm_malloc_gfp(n_pages, sizeof(*pages), GFP_TEMPORARY);
2286                 if (!pages)
2287                         return NULL;
2288         }
2289
2290         for_each_sgt_page(page, sgt_iter, sgt)
2291                 pages[i++] = page;
2292
2293         /* Check that we have the expected number of pages */
2294         GEM_BUG_ON(i != n_pages);
2295
2296         switch (type) {
2297         case I915_MAP_WB:
2298                 pgprot = PAGE_KERNEL;
2299                 break;
2300         case I915_MAP_WC:
2301                 pgprot = pgprot_writecombine(PAGE_KERNEL_IO);
2302                 break;
2303         }
2304         addr = vmap(pages, n_pages, 0, pgprot);
2305
2306         if (pages != stack_pages)
2307                 drm_free_large(pages);
2308
2309         return addr;
2310 }
2311
2312 /* get, pin, and map the pages of the object into kernel space */
2313 void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj,
2314                               enum i915_map_type type)
2315 {
2316         enum i915_map_type has_type;
2317         bool pinned;
2318         void *ptr;
2319         int ret;
2320
2321         lockdep_assert_held(&obj->base.dev->struct_mutex);
2322         GEM_BUG_ON(!i915_gem_object_has_struct_page(obj));
2323
2324         ret = i915_gem_object_get_pages(obj);
2325         if (ret)
2326                 return ERR_PTR(ret);
2327
2328         i915_gem_object_pin_pages(obj);
2329         pinned = obj->pages_pin_count > 1;
2330
2331         ptr = ptr_unpack_bits(obj->mapping, has_type);
2332         if (ptr && has_type != type) {
2333                 if (pinned) {
2334                         ret = -EBUSY;
2335                         goto err;
2336                 }
2337
2338                 if (is_vmalloc_addr(ptr))
2339                         vunmap(ptr);
2340                 else
2341                         kunmap(kmap_to_page(ptr));
2342
2343                 ptr = obj->mapping = NULL;
2344         }
2345
2346         if (!ptr) {
2347                 ptr = i915_gem_object_map(obj, type);
2348                 if (!ptr) {
2349                         ret = -ENOMEM;
2350                         goto err;
2351                 }
2352
2353                 obj->mapping = ptr_pack_bits(ptr, type);
2354         }
2355
2356         return ptr;
2357
2358 err:
2359         i915_gem_object_unpin_pages(obj);
2360         return ERR_PTR(ret);
2361 }
2362
2363 static void
2364 i915_gem_object_retire__write(struct i915_gem_active *active,
2365                               struct drm_i915_gem_request *request)
2366 {
2367         struct drm_i915_gem_object *obj =
2368                 container_of(active, struct drm_i915_gem_object, last_write);
2369
2370         intel_fb_obj_flush(obj, true, ORIGIN_CS);
2371 }
2372
2373 static void
2374 i915_gem_object_retire__read(struct i915_gem_active *active,
2375                              struct drm_i915_gem_request *request)
2376 {
2377         int idx = request->engine->id;
2378         struct drm_i915_gem_object *obj =
2379                 container_of(active, struct drm_i915_gem_object, last_read[idx]);
2380
2381         GEM_BUG_ON(!i915_gem_object_has_active_engine(obj, idx));
2382
2383         i915_gem_object_clear_active(obj, idx);
2384         if (i915_gem_object_is_active(obj))
2385                 return;
2386
2387         /* Bump our place on the bound list to keep it roughly in LRU order
2388          * so that we don't steal from recently used but inactive objects
2389          * (unless we are forced to ofc!)
2390          */
2391         if (obj->bind_count)
2392                 list_move_tail(&obj->global_list,
2393                                &request->i915->mm.bound_list);
2394
2395         i915_gem_object_put(obj);
2396 }
2397
2398 static bool i915_context_is_banned(const struct i915_gem_context *ctx)
2399 {
2400         unsigned long elapsed;
2401
2402         if (ctx->hang_stats.banned)
2403                 return true;
2404
2405         elapsed = get_seconds() - ctx->hang_stats.guilty_ts;
2406         if (ctx->hang_stats.ban_period_seconds &&
2407             elapsed <= ctx->hang_stats.ban_period_seconds) {
2408                 DRM_DEBUG("context hanging too fast, banning!\n");
2409                 return true;
2410         }
2411
2412         return false;
2413 }
2414
2415 static void i915_set_reset_status(struct i915_gem_context *ctx,
2416                                   const bool guilty)
2417 {
2418         struct i915_ctx_hang_stats *hs = &ctx->hang_stats;
2419
2420         if (guilty) {
2421                 hs->banned = i915_context_is_banned(ctx);
2422                 hs->batch_active++;
2423                 hs->guilty_ts = get_seconds();
2424         } else {
2425                 hs->batch_pending++;
2426         }
2427 }
2428
2429 struct drm_i915_gem_request *
2430 i915_gem_find_active_request(struct intel_engine_cs *engine)
2431 {
2432         struct drm_i915_gem_request *request;
2433
2434         /* We are called by the error capture and reset at a random
2435          * point in time. In particular, note that neither is crucially
2436          * ordered with an interrupt. After a hang, the GPU is dead and we
2437          * assume that no more writes can happen (we waited long enough for
2438          * all writes that were in transaction to be flushed) - adding an
2439          * extra delay for a recent interrupt is pointless. Hence, we do
2440          * not need an engine->irq_seqno_barrier() before the seqno reads.
2441          */
2442         list_for_each_entry(request, &engine->request_list, link) {
2443                 if (i915_gem_request_completed(request))
2444                         continue;
2445
2446                 return request;
2447         }
2448
2449         return NULL;
2450 }
2451
2452 static void i915_gem_reset_engine_status(struct intel_engine_cs *engine)
2453 {
2454         struct drm_i915_gem_request *request;
2455         bool ring_hung;
2456
2457         request = i915_gem_find_active_request(engine);
2458         if (request == NULL)
2459                 return;
2460
2461         ring_hung = engine->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG;
2462
2463         i915_set_reset_status(request->ctx, ring_hung);
2464         list_for_each_entry_continue(request, &engine->request_list, link)
2465                 i915_set_reset_status(request->ctx, false);
2466 }
2467
2468 static void i915_gem_reset_engine_cleanup(struct intel_engine_cs *engine)
2469 {
2470         struct drm_i915_gem_request *request;
2471         struct intel_ring *ring;
2472
2473         /* Mark all pending requests as complete so that any concurrent
2474          * (lockless) lookup doesn't try and wait upon the request as we
2475          * reset it.
2476          */
2477         intel_engine_init_seqno(engine, engine->last_submitted_seqno);
2478
2479         /*
2480          * Clear the execlists queue up before freeing the requests, as those
2481          * are the ones that keep the context and ringbuffer backing objects
2482          * pinned in place.
2483          */
2484
2485         if (i915.enable_execlists) {
2486                 /* Ensure irq handler finishes or is cancelled. */
2487                 tasklet_kill(&engine->irq_tasklet);
2488
2489                 intel_execlists_cancel_requests(engine);
2490         }
2491
2492         /*
2493          * We must free the requests after all the corresponding objects have
2494          * been moved off active lists. Which is the same order as the normal
2495          * retire_requests function does. This is important if object hold
2496          * implicit references on things like e.g. ppgtt address spaces through
2497          * the request.
2498          */
2499         request = i915_gem_active_raw(&engine->last_request,
2500                                       &engine->i915->drm.struct_mutex);
2501         if (request)
2502                 i915_gem_request_retire_upto(request);
2503         GEM_BUG_ON(intel_engine_is_active(engine));
2504
2505         /* Having flushed all requests from all queues, we know that all
2506          * ringbuffers must now be empty. However, since we do not reclaim
2507          * all space when retiring the request (to prevent HEADs colliding
2508          * with rapid ringbuffer wraparound) the amount of available space
2509          * upon reset is less than when we start. Do one more pass over
2510          * all the ringbuffers to reset last_retired_head.
2511          */
2512         list_for_each_entry(ring, &engine->buffers, link) {
2513                 ring->last_retired_head = ring->tail;
2514                 intel_ring_update_space(ring);
2515         }
2516
2517         engine->i915->gt.active_engines &= ~intel_engine_flag(engine);
2518 }
2519
2520 void i915_gem_reset(struct drm_device *dev)
2521 {
2522         struct drm_i915_private *dev_priv = to_i915(dev);
2523         struct intel_engine_cs *engine;
2524
2525         /*
2526          * Before we free the objects from the requests, we need to inspect
2527          * them for finding the guilty party. As the requests only borrow
2528          * their reference to the objects, the inspection must be done first.
2529          */
2530         for_each_engine(engine, dev_priv)
2531                 i915_gem_reset_engine_status(engine);
2532
2533         for_each_engine(engine, dev_priv)
2534                 i915_gem_reset_engine_cleanup(engine);
2535         mod_delayed_work(dev_priv->wq, &dev_priv->gt.idle_work, 0);
2536
2537         i915_gem_context_reset(dev);
2538
2539         i915_gem_restore_fences(dev);
2540 }
2541
2542 static void
2543 i915_gem_retire_work_handler(struct work_struct *work)
2544 {
2545         struct drm_i915_private *dev_priv =
2546                 container_of(work, typeof(*dev_priv), gt.retire_work.work);
2547         struct drm_device *dev = &dev_priv->drm;
2548
2549         /* Come back later if the device is busy... */
2550         if (mutex_trylock(&dev->struct_mutex)) {
2551                 i915_gem_retire_requests(dev_priv);
2552                 mutex_unlock(&dev->struct_mutex);
2553         }
2554
2555         /* Keep the retire handler running until we are finally idle.
2556          * We do not need to do this test under locking as in the worst-case
2557          * we queue the retire worker once too often.
2558          */
2559         if (READ_ONCE(dev_priv->gt.awake)) {
2560                 i915_queue_hangcheck(dev_priv);
2561                 queue_delayed_work(dev_priv->wq,
2562                                    &dev_priv->gt.retire_work,
2563                                    round_jiffies_up_relative(HZ));
2564         }
2565 }
2566
2567 static void
2568 i915_gem_idle_work_handler(struct work_struct *work)
2569 {
2570         struct drm_i915_private *dev_priv =
2571                 container_of(work, typeof(*dev_priv), gt.idle_work.work);
2572         struct drm_device *dev = &dev_priv->drm;
2573         struct intel_engine_cs *engine;
2574         bool rearm_hangcheck;
2575
2576         if (!READ_ONCE(dev_priv->gt.awake))
2577                 return;
2578
2579         if (READ_ONCE(dev_priv->gt.active_engines))
2580                 return;
2581
2582         rearm_hangcheck =
2583                 cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work);
2584
2585         if (!mutex_trylock(&dev->struct_mutex)) {
2586                 /* Currently busy, come back later */
2587                 mod_delayed_work(dev_priv->wq,
2588                                  &dev_priv->gt.idle_work,
2589                                  msecs_to_jiffies(50));
2590                 goto out_rearm;
2591         }
2592
2593         if (dev_priv->gt.active_engines)
2594                 goto out_unlock;
2595
2596         for_each_engine(engine, dev_priv)
2597                 i915_gem_batch_pool_fini(&engine->batch_pool);
2598
2599         GEM_BUG_ON(!dev_priv->gt.awake);
2600         dev_priv->gt.awake = false;
2601         rearm_hangcheck = false;
2602
2603         if (INTEL_GEN(dev_priv) >= 6)
2604                 gen6_rps_idle(dev_priv);
2605         intel_runtime_pm_put(dev_priv);
2606 out_unlock:
2607         mutex_unlock(&dev->struct_mutex);
2608
2609 out_rearm:
2610         if (rearm_hangcheck) {
2611                 GEM_BUG_ON(!dev_priv->gt.awake);
2612                 i915_queue_hangcheck(dev_priv);
2613         }
2614 }
2615
2616 void i915_gem_close_object(struct drm_gem_object *gem, struct drm_file *file)
2617 {
2618         struct drm_i915_gem_object *obj = to_intel_bo(gem);
2619         struct drm_i915_file_private *fpriv = file->driver_priv;
2620         struct i915_vma *vma, *vn;
2621
2622         mutex_lock(&obj->base.dev->struct_mutex);
2623         list_for_each_entry_safe(vma, vn, &obj->vma_list, obj_link)
2624                 if (vma->vm->file == fpriv)
2625                         i915_vma_close(vma);
2626         mutex_unlock(&obj->base.dev->struct_mutex);
2627 }
2628
2629 /**
2630  * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
2631  * @dev: drm device pointer
2632  * @data: ioctl data blob
2633  * @file: drm file pointer
2634  *
2635  * Returns 0 if successful, else an error is returned with the remaining time in
2636  * the timeout parameter.
2637  *  -ETIME: object is still busy after timeout
2638  *  -ERESTARTSYS: signal interrupted the wait
2639  *  -ENONENT: object doesn't exist
2640  * Also possible, but rare:
2641  *  -EAGAIN: GPU wedged
2642  *  -ENOMEM: damn
2643  *  -ENODEV: Internal IRQ fail
2644  *  -E?: The add request failed
2645  *
2646  * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
2647  * non-zero timeout parameter the wait ioctl will wait for the given number of
2648  * nanoseconds on an object becoming unbusy. Since the wait itself does so
2649  * without holding struct_mutex the object may become re-busied before this
2650  * function completes. A similar but shorter * race condition exists in the busy
2651  * ioctl
2652  */
2653 int
2654 i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
2655 {
2656         struct drm_i915_gem_wait *args = data;
2657         struct intel_rps_client *rps = to_rps_client(file);
2658         struct drm_i915_gem_object *obj;
2659         unsigned long active;
2660         int idx, ret = 0;
2661
2662         if (args->flags != 0)
2663                 return -EINVAL;
2664
2665         obj = i915_gem_object_lookup(file, args->bo_handle);
2666         if (!obj)
2667                 return -ENOENT;
2668
2669         active = __I915_BO_ACTIVE(obj);
2670         for_each_active(active, idx) {
2671                 s64 *timeout = args->timeout_ns >= 0 ? &args->timeout_ns : NULL;
2672                 ret = i915_gem_active_wait_unlocked(&obj->last_read[idx], true,
2673                                                     timeout, rps);
2674                 if (ret)
2675                         break;
2676         }
2677
2678         i915_gem_object_put_unlocked(obj);
2679         return ret;
2680 }
2681
2682 static int
2683 __i915_gem_object_sync(struct drm_i915_gem_request *to,
2684                        struct drm_i915_gem_request *from)
2685 {
2686         int ret;
2687
2688         if (to->engine == from->engine)
2689                 return 0;
2690
2691         if (!i915.semaphores) {
2692                 ret = i915_wait_request(from,
2693                                         from->i915->mm.interruptible,
2694                                         NULL,
2695                                         NO_WAITBOOST);
2696                 if (ret)
2697                         return ret;
2698         } else {
2699                 int idx = intel_engine_sync_index(from->engine, to->engine);
2700                 if (from->fence.seqno <= from->engine->semaphore.sync_seqno[idx])
2701                         return 0;
2702
2703                 trace_i915_gem_ring_sync_to(to, from);
2704                 ret = to->engine->semaphore.sync_to(to, from);
2705                 if (ret)
2706                         return ret;
2707
2708                 from->engine->semaphore.sync_seqno[idx] = from->fence.seqno;
2709         }
2710
2711         return 0;
2712 }
2713
2714 /**
2715  * i915_gem_object_sync - sync an object to a ring.
2716  *
2717  * @obj: object which may be in use on another ring.
2718  * @to: request we are wishing to use
2719  *
2720  * This code is meant to abstract object synchronization with the GPU.
2721  * Conceptually we serialise writes between engines inside the GPU.
2722  * We only allow one engine to write into a buffer at any time, but
2723  * multiple readers. To ensure each has a coherent view of memory, we must:
2724  *
2725  * - If there is an outstanding write request to the object, the new
2726  *   request must wait for it to complete (either CPU or in hw, requests
2727  *   on the same ring will be naturally ordered).
2728  *
2729  * - If we are a write request (pending_write_domain is set), the new
2730  *   request must wait for outstanding read requests to complete.
2731  *
2732  * Returns 0 if successful, else propagates up the lower layer error.
2733  */
2734 int
2735 i915_gem_object_sync(struct drm_i915_gem_object *obj,
2736                      struct drm_i915_gem_request *to)
2737 {
2738         struct i915_gem_active *active;
2739         unsigned long active_mask;
2740         int idx;
2741
2742         lockdep_assert_held(&obj->base.dev->struct_mutex);
2743
2744         active_mask = i915_gem_object_get_active(obj);
2745         if (!active_mask)
2746                 return 0;
2747
2748         if (obj->base.pending_write_domain) {
2749                 active = obj->last_read;
2750         } else {
2751                 active_mask = 1;
2752                 active = &obj->last_write;
2753         }
2754
2755         for_each_active(active_mask, idx) {
2756                 struct drm_i915_gem_request *request;
2757                 int ret;
2758
2759                 request = i915_gem_active_peek(&active[idx],
2760                                                &obj->base.dev->struct_mutex);
2761                 if (!request)
2762                         continue;
2763
2764                 ret = __i915_gem_object_sync(to, request);
2765                 if (ret)
2766                         return ret;
2767         }
2768
2769         return 0;
2770 }
2771
2772 static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
2773 {
2774         u32 old_write_domain, old_read_domains;
2775
2776         /* Force a pagefault for domain tracking on next user access */
2777         i915_gem_release_mmap(obj);
2778
2779         if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
2780                 return;
2781
2782         old_read_domains = obj->base.read_domains;
2783         old_write_domain = obj->base.write_domain;
2784
2785         obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
2786         obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
2787
2788         trace_i915_gem_object_change_domain(obj,
2789                                             old_read_domains,
2790                                             old_write_domain);
2791 }
2792
2793 static void __i915_vma_iounmap(struct i915_vma *vma)
2794 {
2795         GEM_BUG_ON(i915_vma_is_pinned(vma));
2796
2797         if (vma->iomap == NULL)
2798                 return;
2799
2800         io_mapping_unmap(vma->iomap);
2801         vma->iomap = NULL;
2802 }
2803
2804 int i915_vma_unbind(struct i915_vma *vma)
2805 {
2806         struct drm_i915_gem_object *obj = vma->obj;
2807         unsigned long active;
2808         int ret;
2809
2810         /* First wait upon any activity as retiring the request may
2811          * have side-effects such as unpinning or even unbinding this vma.
2812          */
2813         active = i915_vma_get_active(vma);
2814         if (active) {
2815                 int idx;
2816
2817                 /* When a closed VMA is retired, it is unbound - eek.
2818                  * In order to prevent it from being recursively closed,
2819                  * take a pin on the vma so that the second unbind is
2820                  * aborted.
2821                  */
2822                 __i915_vma_pin(vma);
2823
2824                 for_each_active(active, idx) {
2825                         ret = i915_gem_active_retire(&vma->last_read[idx],
2826                                                    &vma->vm->dev->struct_mutex);
2827                         if (ret)
2828                                 break;
2829                 }
2830
2831                 __i915_vma_unpin(vma);
2832                 if (ret)
2833                         return ret;
2834
2835                 GEM_BUG_ON(i915_vma_is_active(vma));
2836         }
2837
2838         if (i915_vma_is_pinned(vma))
2839                 return -EBUSY;
2840
2841         if (!drm_mm_node_allocated(&vma->node))
2842                 goto destroy;
2843
2844         GEM_BUG_ON(obj->bind_count == 0);
2845         GEM_BUG_ON(!obj->pages);
2846
2847         if (i915_vma_is_ggtt(vma) &&
2848             vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL) {
2849                 i915_gem_object_finish_gtt(obj);
2850
2851                 /* release the fence reg _after_ flushing */
2852                 ret = i915_gem_object_put_fence(obj);
2853                 if (ret)
2854                         return ret;
2855
2856                 __i915_vma_iounmap(vma);
2857         }
2858
2859         if (likely(!vma->vm->closed)) {
2860                 trace_i915_vma_unbind(vma);
2861                 vma->vm->unbind_vma(vma);
2862         }
2863         vma->flags &= ~(I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND);
2864
2865         drm_mm_remove_node(&vma->node);
2866         list_move_tail(&vma->vm_link, &vma->vm->unbound_list);
2867
2868         if (i915_vma_is_ggtt(vma)) {
2869                 if (vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL) {
2870                         obj->map_and_fenceable = false;
2871                 } else if (vma->ggtt_view.pages) {
2872                         sg_free_table(vma->ggtt_view.pages);
2873                         kfree(vma->ggtt_view.pages);
2874                 }
2875                 vma->ggtt_view.pages = NULL;
2876         }
2877
2878         /* Since the unbound list is global, only move to that list if
2879          * no more VMAs exist. */
2880         if (--obj->bind_count == 0)
2881                 list_move_tail(&obj->global_list,
2882                                &to_i915(obj->base.dev)->mm.unbound_list);
2883
2884         /* And finally now the object is completely decoupled from this vma,
2885          * we can drop its hold on the backing storage and allow it to be
2886          * reaped by the shrinker.
2887          */
2888         i915_gem_object_unpin_pages(obj);
2889
2890 destroy:
2891         if (unlikely(i915_vma_is_closed(vma)))
2892                 i915_vma_destroy(vma);
2893
2894         return 0;
2895 }
2896
2897 int i915_gem_wait_for_idle(struct drm_i915_private *dev_priv,
2898                            bool interruptible)
2899 {
2900         struct intel_engine_cs *engine;
2901         int ret;
2902
2903         for_each_engine(engine, dev_priv) {
2904                 if (engine->last_context == NULL)
2905                         continue;
2906
2907                 ret = intel_engine_idle(engine, interruptible);
2908                 if (ret)
2909                         return ret;
2910         }
2911
2912         return 0;
2913 }
2914
2915 static bool i915_gem_valid_gtt_space(struct i915_vma *vma,
2916                                      unsigned long cache_level)
2917 {
2918         struct drm_mm_node *gtt_space = &vma->node;
2919         struct drm_mm_node *other;
2920
2921         /*
2922          * On some machines we have to be careful when putting differing types
2923          * of snoopable memory together to avoid the prefetcher crossing memory
2924          * domains and dying. During vm initialisation, we decide whether or not
2925          * these constraints apply and set the drm_mm.color_adjust
2926          * appropriately.
2927          */
2928         if (vma->vm->mm.color_adjust == NULL)
2929                 return true;
2930
2931         if (!drm_mm_node_allocated(gtt_space))
2932                 return true;
2933
2934         if (list_empty(&gtt_space->node_list))
2935                 return true;
2936
2937         other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list);
2938         if (other->allocated && !other->hole_follows && other->color != cache_level)
2939                 return false;
2940
2941         other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list);
2942         if (other->allocated && !gtt_space->hole_follows && other->color != cache_level)
2943                 return false;
2944
2945         return true;
2946 }
2947
2948 /**
2949  * i915_vma_insert - finds a slot for the vma in its address space
2950  * @vma: the vma
2951  * @size: requested size in bytes (can be larger than the VMA)
2952  * @alignment: required alignment
2953  * @flags: mask of PIN_* flags to use
2954  *
2955  * First we try to allocate some free space that meets the requirements for
2956  * the VMA. Failiing that, if the flags permit, it will evict an old VMA,
2957  * preferrably the oldest idle entry to make room for the new VMA.
2958  *
2959  * Returns:
2960  * 0 on success, negative error code otherwise.
2961  */
2962 static int
2963 i915_vma_insert(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
2964 {
2965         struct drm_i915_private *dev_priv = to_i915(vma->vm->dev);
2966         struct drm_i915_gem_object *obj = vma->obj;
2967         u64 start, end;
2968         u64 min_alignment;
2969         int ret;
2970
2971         GEM_BUG_ON(vma->flags & (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND));
2972         GEM_BUG_ON(drm_mm_node_allocated(&vma->node));
2973
2974         size = max(size, vma->size);
2975         if (flags & PIN_MAPPABLE)
2976                 size = i915_gem_get_ggtt_size(dev_priv, size,
2977                                               i915_gem_object_get_tiling(obj));
2978
2979         min_alignment =
2980                 i915_gem_get_ggtt_alignment(dev_priv, size,
2981                                             i915_gem_object_get_tiling(obj),
2982                                             flags & PIN_MAPPABLE);
2983         if (alignment == 0)
2984                 alignment = min_alignment;
2985         if (alignment & (min_alignment - 1)) {
2986                 DRM_DEBUG("Invalid object alignment requested %llu, minimum %llu\n",
2987                           alignment, min_alignment);
2988                 return -EINVAL;
2989         }
2990
2991         start = flags & PIN_OFFSET_BIAS ? flags & PIN_OFFSET_MASK : 0;
2992
2993         end = vma->vm->total;
2994         if (flags & PIN_MAPPABLE)
2995                 end = min_t(u64, end, dev_priv->ggtt.mappable_end);
2996         if (flags & PIN_ZONE_4G)
2997                 end = min_t(u64, end, (1ULL << 32) - PAGE_SIZE);
2998
2999         /* If binding the object/GGTT view requires more space than the entire
3000          * aperture has, reject it early before evicting everything in a vain
3001          * attempt to find space.
3002          */
3003         if (size > end) {
3004                 DRM_DEBUG("Attempting to bind an object larger than the aperture: request=%llu [object=%zd] > %s aperture=%llu\n",
3005                           size, obj->base.size,
3006                           flags & PIN_MAPPABLE ? "mappable" : "total",
3007                           end);
3008                 return -E2BIG;
3009         }
3010
3011         ret = i915_gem_object_get_pages(obj);
3012         if (ret)
3013                 return ret;
3014
3015         i915_gem_object_pin_pages(obj);
3016
3017         if (flags & PIN_OFFSET_FIXED) {
3018                 u64 offset = flags & PIN_OFFSET_MASK;
3019                 if (offset & (alignment - 1) || offset > end - size) {
3020                         ret = -EINVAL;
3021                         goto err_unpin;
3022                 }
3023
3024                 vma->node.start = offset;
3025                 vma->node.size = size;
3026                 vma->node.color = obj->cache_level;
3027                 ret = drm_mm_reserve_node(&vma->vm->mm, &vma->node);
3028                 if (ret) {
3029                         ret = i915_gem_evict_for_vma(vma);
3030                         if (ret == 0)
3031                                 ret = drm_mm_reserve_node(&vma->vm->mm, &vma->node);
3032                         if (ret)
3033                                 goto err_unpin;
3034                 }
3035         } else {
3036                 u32 search_flag, alloc_flag;
3037
3038                 if (flags & PIN_HIGH) {
3039                         search_flag = DRM_MM_SEARCH_BELOW;
3040                         alloc_flag = DRM_MM_CREATE_TOP;
3041                 } else {
3042                         search_flag = DRM_MM_SEARCH_DEFAULT;
3043                         alloc_flag = DRM_MM_CREATE_DEFAULT;
3044                 }
3045
3046                 /* We only allocate in PAGE_SIZE/GTT_PAGE_SIZE (4096) chunks,
3047                  * so we know that we always have a minimum alignment of 4096.
3048                  * The drm_mm range manager is optimised to return results
3049                  * with zero alignment, so where possible use the optimal
3050                  * path.
3051                  */
3052                 if (alignment <= 4096)
3053                         alignment = 0;
3054
3055 search_free:
3056                 ret = drm_mm_insert_node_in_range_generic(&vma->vm->mm,
3057                                                           &vma->node,
3058                                                           size, alignment,
3059                                                           obj->cache_level,
3060                                                           start, end,
3061                                                           search_flag,
3062                                                           alloc_flag);
3063                 if (ret) {
3064                         ret = i915_gem_evict_something(vma->vm, size, alignment,
3065                                                        obj->cache_level,
3066                                                        start, end,
3067                                                        flags);
3068                         if (ret == 0)
3069                                 goto search_free;
3070
3071                         goto err_unpin;
3072                 }
3073         }
3074         GEM_BUG_ON(!i915_gem_valid_gtt_space(vma, obj->cache_level));
3075
3076         list_move_tail(&obj->global_list, &dev_priv->mm.bound_list);
3077         list_move_tail(&vma->vm_link, &vma->vm->inactive_list);
3078         obj->bind_count++;
3079
3080         return 0;
3081
3082 err_unpin:
3083         i915_gem_object_unpin_pages(obj);
3084         return ret;
3085 }
3086
3087 bool
3088 i915_gem_clflush_object(struct drm_i915_gem_object *obj,
3089                         bool force)
3090 {
3091         /* If we don't have a page list set up, then we're not pinned
3092          * to GPU, and we can ignore the cache flush because it'll happen
3093          * again at bind time.
3094          */
3095         if (obj->pages == NULL)
3096                 return false;
3097
3098         /*
3099          * Stolen memory is always coherent with the GPU as it is explicitly
3100          * marked as wc by the system, or the system is cache-coherent.
3101          */
3102         if (obj->stolen || obj->phys_handle)
3103                 return false;
3104
3105         /* If the GPU is snooping the contents of the CPU cache,
3106          * we do not need to manually clear the CPU cache lines.  However,
3107          * the caches are only snooped when the render cache is
3108          * flushed/invalidated.  As we always have to emit invalidations
3109          * and flushes when moving into and out of the RENDER domain, correct
3110          * snooping behaviour occurs naturally as the result of our domain
3111          * tracking.
3112          */
3113         if (!force && cpu_cache_is_coherent(obj->base.dev, obj->cache_level)) {
3114                 obj->cache_dirty = true;
3115                 return false;
3116         }
3117
3118         trace_i915_gem_object_clflush(obj);
3119         drm_clflush_sg(obj->pages);
3120         obj->cache_dirty = false;
3121
3122         return true;
3123 }
3124
3125 /** Flushes the GTT write domain for the object if it's dirty. */
3126 static void
3127 i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
3128 {
3129         uint32_t old_write_domain;
3130
3131         if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
3132                 return;
3133
3134         /* No actual flushing is required for the GTT write domain.  Writes
3135          * to it immediately go to main memory as far as we know, so there's
3136          * no chipset flush.  It also doesn't land in render cache.
3137          *
3138          * However, we do have to enforce the order so that all writes through
3139          * the GTT land before any writes to the device, such as updates to
3140          * the GATT itself.
3141          */
3142         wmb();
3143
3144         old_write_domain = obj->base.write_domain;
3145         obj->base.write_domain = 0;
3146
3147         intel_fb_obj_flush(obj, false, ORIGIN_GTT);
3148
3149         trace_i915_gem_object_change_domain(obj,
3150                                             obj->base.read_domains,
3151                                             old_write_domain);
3152 }
3153
3154 /** Flushes the CPU write domain for the object if it's dirty. */
3155 static void
3156 i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
3157 {
3158         uint32_t old_write_domain;
3159
3160         if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
3161                 return;
3162
3163         if (i915_gem_clflush_object(obj, obj->pin_display))
3164                 i915_gem_chipset_flush(to_i915(obj->base.dev));
3165
3166         old_write_domain = obj->base.write_domain;
3167         obj->base.write_domain = 0;
3168
3169         intel_fb_obj_flush(obj, false, ORIGIN_CPU);
3170
3171         trace_i915_gem_object_change_domain(obj,
3172                                             obj->base.read_domains,
3173                                             old_write_domain);
3174 }
3175
3176 /**
3177  * Moves a single object to the GTT read, and possibly write domain.
3178  * @obj: object to act on
3179  * @write: ask for write access or read only
3180  *
3181  * This function returns when the move is complete, including waiting on
3182  * flushes to occur.
3183  */
3184 int
3185 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
3186 {
3187         uint32_t old_write_domain, old_read_domains;
3188         struct i915_vma *vma;
3189         int ret;
3190
3191         ret = i915_gem_object_wait_rendering(obj, !write);
3192         if (ret)
3193                 return ret;
3194
3195         if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
3196                 return 0;
3197
3198         /* Flush and acquire obj->pages so that we are coherent through
3199          * direct access in memory with previous cached writes through
3200          * shmemfs and that our cache domain tracking remains valid.
3201          * For example, if the obj->filp was moved to swap without us
3202          * being notified and releasing the pages, we would mistakenly
3203          * continue to assume that the obj remained out of the CPU cached
3204          * domain.
3205          */
3206         ret = i915_gem_object_get_pages(obj);
3207         if (ret)
3208                 return ret;
3209
3210         i915_gem_object_flush_cpu_write_domain(obj);
3211
3212         /* Serialise direct access to this object with the barriers for
3213          * coherent writes from the GPU, by effectively invalidating the
3214          * GTT domain upon first access.
3215          */
3216         if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
3217                 mb();
3218
3219         old_write_domain = obj->base.write_domain;
3220         old_read_domains = obj->base.read_domains;
3221
3222         /* It should now be out of any other write domains, and we can update
3223          * the domain values for our changes.
3224          */
3225         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
3226         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
3227         if (write) {
3228                 obj->base.read_domains = I915_GEM_DOMAIN_GTT;
3229                 obj->base.write_domain = I915_GEM_DOMAIN_GTT;
3230                 obj->dirty = 1;
3231         }
3232
3233         trace_i915_gem_object_change_domain(obj,
3234                                             old_read_domains,
3235                                             old_write_domain);
3236
3237         /* And bump the LRU for this access */
3238         vma = i915_gem_obj_to_ggtt(obj);
3239         if (vma &&
3240             drm_mm_node_allocated(&vma->node) &&
3241             !i915_vma_is_active(vma))
3242                 list_move_tail(&vma->vm_link, &vma->vm->inactive_list);
3243
3244         return 0;
3245 }
3246
3247 /**
3248  * Changes the cache-level of an object across all VMA.
3249  * @obj: object to act on
3250  * @cache_level: new cache level to set for the object
3251  *
3252  * After this function returns, the object will be in the new cache-level
3253  * across all GTT and the contents of the backing storage will be coherent,
3254  * with respect to the new cache-level. In order to keep the backing storage
3255  * coherent for all users, we only allow a single cache level to be set
3256  * globally on the object and prevent it from being changed whilst the
3257  * hardware is reading from the object. That is if the object is currently
3258  * on the scanout it will be set to uncached (or equivalent display
3259  * cache coherency) and all non-MOCS GPU access will also be uncached so
3260  * that all direct access to the scanout remains coherent.
3261  */
3262 int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
3263                                     enum i915_cache_level cache_level)
3264 {
3265         struct i915_vma *vma;
3266         int ret = 0;
3267
3268         if (obj->cache_level == cache_level)
3269                 goto out;
3270
3271         /* Inspect the list of currently bound VMA and unbind any that would
3272          * be invalid given the new cache-level. This is principally to
3273          * catch the issue of the CS prefetch crossing page boundaries and
3274          * reading an invalid PTE on older architectures.
3275          */
3276 restart:
3277         list_for_each_entry(vma, &obj->vma_list, obj_link) {
3278                 if (!drm_mm_node_allocated(&vma->node))
3279                         continue;
3280
3281                 if (i915_vma_is_pinned(vma)) {
3282                         DRM_DEBUG("can not change the cache level of pinned objects\n");
3283                         return -EBUSY;
3284                 }
3285
3286                 if (i915_gem_valid_gtt_space(vma, cache_level))
3287                         continue;
3288
3289                 ret = i915_vma_unbind(vma);
3290                 if (ret)
3291                         return ret;
3292
3293                 /* As unbinding may affect other elements in the
3294                  * obj->vma_list (due to side-effects from retiring
3295                  * an active vma), play safe and restart the iterator.
3296                  */
3297                 goto restart;
3298         }
3299
3300         /* We can reuse the existing drm_mm nodes but need to change the
3301          * cache-level on the PTE. We could simply unbind them all and
3302          * rebind with the correct cache-level on next use. However since
3303          * we already have a valid slot, dma mapping, pages etc, we may as
3304          * rewrite the PTE in the belief that doing so tramples upon less
3305          * state and so involves less work.
3306          */
3307         if (obj->bind_count) {
3308                 /* Before we change the PTE, the GPU must not be accessing it.
3309                  * If we wait upon the object, we know that all the bound
3310                  * VMA are no longer active.
3311                  */
3312                 ret = i915_gem_object_wait_rendering(obj, false);
3313                 if (ret)
3314                         return ret;
3315
3316                 if (!HAS_LLC(obj->base.dev) && cache_level != I915_CACHE_NONE) {
3317                         /* Access to snoopable pages through the GTT is
3318                          * incoherent and on some machines causes a hard
3319                          * lockup. Relinquish the CPU mmaping to force
3320                          * userspace to refault in the pages and we can
3321                          * then double check if the GTT mapping is still
3322                          * valid for that pointer access.
3323                          */
3324                         i915_gem_release_mmap(obj);
3325
3326                         /* As we no longer need a fence for GTT access,
3327                          * we can relinquish it now (and so prevent having
3328                          * to steal a fence from someone else on the next
3329                          * fence request). Note GPU activity would have
3330                          * dropped the fence as all snoopable access is
3331                          * supposed to be linear.
3332                          */
3333                         ret = i915_gem_object_put_fence(obj);
3334                         if (ret)
3335                                 return ret;
3336                 } else {
3337                         /* We either have incoherent backing store and
3338                          * so no GTT access or the architecture is fully
3339                          * coherent. In such cases, existing GTT mmaps
3340                          * ignore the cache bit in the PTE and we can
3341                          * rewrite it without confusing the GPU or having
3342                          * to force userspace to fault back in its mmaps.
3343                          */
3344                 }
3345
3346                 list_for_each_entry(vma, &obj->vma_list, obj_link) {
3347                         if (!drm_mm_node_allocated(&vma->node))
3348                                 continue;
3349
3350                         ret = i915_vma_bind(vma, cache_level, PIN_UPDATE);
3351                         if (ret)
3352                                 return ret;
3353                 }
3354         }
3355
3356         list_for_each_entry(vma, &obj->vma_list, obj_link)
3357                 vma->node.color = cache_level;
3358         obj->cache_level = cache_level;
3359
3360 out:
3361         /* Flush the dirty CPU caches to the backing storage so that the
3362          * object is now coherent at its new cache level (with respect
3363          * to the access domain).
3364          */
3365         if (obj->cache_dirty && cpu_write_needs_clflush(obj)) {
3366                 if (i915_gem_clflush_object(obj, true))
3367                         i915_gem_chipset_flush(to_i915(obj->base.dev));
3368         }
3369
3370         return 0;
3371 }
3372
3373 int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data,
3374                                struct drm_file *file)
3375 {
3376         struct drm_i915_gem_caching *args = data;
3377         struct drm_i915_gem_object *obj;
3378
3379         obj = i915_gem_object_lookup(file, args->handle);
3380         if (!obj)
3381                 return -ENOENT;
3382
3383         switch (obj->cache_level) {
3384         case I915_CACHE_LLC:
3385         case I915_CACHE_L3_LLC:
3386                 args->caching = I915_CACHING_CACHED;
3387                 break;
3388
3389         case I915_CACHE_WT:
3390                 args->caching = I915_CACHING_DISPLAY;
3391                 break;
3392
3393         default:
3394                 args->caching = I915_CACHING_NONE;
3395                 break;
3396         }
3397
3398         i915_gem_object_put_unlocked(obj);
3399         return 0;
3400 }
3401
3402 int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
3403                                struct drm_file *file)
3404 {
3405         struct drm_i915_private *dev_priv = to_i915(dev);
3406         struct drm_i915_gem_caching *args = data;
3407         struct drm_i915_gem_object *obj;
3408         enum i915_cache_level level;
3409         int ret;
3410
3411         switch (args->caching) {
3412         case I915_CACHING_NONE:
3413                 level = I915_CACHE_NONE;
3414                 break;
3415         case I915_CACHING_CACHED:
3416                 /*
3417                  * Due to a HW issue on BXT A stepping, GPU stores via a
3418                  * snooped mapping may leave stale data in a corresponding CPU
3419                  * cacheline, whereas normally such cachelines would get
3420                  * invalidated.
3421                  */
3422                 if (!HAS_LLC(dev) && !HAS_SNOOP(dev))
3423                         return -ENODEV;
3424
3425                 level = I915_CACHE_LLC;
3426                 break;
3427         case I915_CACHING_DISPLAY:
3428                 level = HAS_WT(dev) ? I915_CACHE_WT : I915_CACHE_NONE;
3429                 break;
3430         default:
3431                 return -EINVAL;
3432         }
3433
3434         intel_runtime_pm_get(dev_priv);
3435
3436         ret = i915_mutex_lock_interruptible(dev);
3437         if (ret)
3438                 goto rpm_put;
3439
3440         obj = i915_gem_object_lookup(file, args->handle);
3441         if (!obj) {
3442                 ret = -ENOENT;
3443                 goto unlock;
3444         }
3445
3446         ret = i915_gem_object_set_cache_level(obj, level);
3447
3448         i915_gem_object_put(obj);
3449 unlock:
3450         mutex_unlock(&dev->struct_mutex);
3451 rpm_put:
3452         intel_runtime_pm_put(dev_priv);
3453
3454         return ret;
3455 }
3456
3457 /*
3458  * Prepare buffer for display plane (scanout, cursors, etc).
3459  * Can be called from an uninterruptible phase (modesetting) and allows
3460  * any flushes to be pipelined (for pageflips).
3461  */
3462 int
3463 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
3464                                      u32 alignment,
3465                                      const struct i915_ggtt_view *view)
3466 {
3467         u32 old_read_domains, old_write_domain;
3468         int ret;
3469
3470         /* Mark the pin_display early so that we account for the
3471          * display coherency whilst setting up the cache domains.
3472          */
3473         obj->pin_display++;
3474
3475         /* The display engine is not coherent with the LLC cache on gen6.  As
3476          * a result, we make sure that the pinning that is about to occur is
3477          * done with uncached PTEs. This is lowest common denominator for all
3478          * chipsets.
3479          *
3480          * However for gen6+, we could do better by using the GFDT bit instead
3481          * of uncaching, which would allow us to flush all the LLC-cached data
3482          * with that bit in the PTE to main memory with just one PIPE_CONTROL.
3483          */
3484         ret = i915_gem_object_set_cache_level(obj,
3485                                               HAS_WT(obj->base.dev) ? I915_CACHE_WT : I915_CACHE_NONE);
3486         if (ret)
3487                 goto err_unpin_display;
3488
3489         /* As the user may map the buffer once pinned in the display plane
3490          * (e.g. libkms for the bootup splash), we have to ensure that we
3491          * always use map_and_fenceable for all scanout buffers.
3492          */
3493         ret = i915_gem_object_ggtt_pin(obj, view, 0, alignment,
3494                                        view->type == I915_GGTT_VIEW_NORMAL ?
3495                                        PIN_MAPPABLE : 0);
3496         if (ret)
3497                 goto err_unpin_display;
3498
3499         i915_gem_object_flush_cpu_write_domain(obj);
3500
3501         old_write_domain = obj->base.write_domain;
3502         old_read_domains = obj->base.read_domains;
3503
3504         /* It should now be out of any other write domains, and we can update
3505          * the domain values for our changes.
3506          */
3507         obj->base.write_domain = 0;
3508         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
3509
3510         trace_i915_gem_object_change_domain(obj,
3511                                             old_read_domains,
3512                                             old_write_domain);
3513
3514         return 0;
3515
3516 err_unpin_display:
3517         obj->pin_display--;
3518         return ret;
3519 }
3520
3521 void
3522 i915_gem_object_unpin_from_display_plane(struct drm_i915_gem_object *obj,
3523                                          const struct i915_ggtt_view *view)
3524 {
3525         if (WARN_ON(obj->pin_display == 0))
3526                 return;
3527
3528         i915_gem_object_ggtt_unpin_view(obj, view);
3529
3530         obj->pin_display--;
3531 }
3532
3533 /**
3534  * Moves a single object to the CPU read, and possibly write domain.
3535  * @obj: object to act on
3536  * @write: requesting write or read-only access
3537  *
3538  * This function returns when the move is complete, including waiting on
3539  * flushes to occur.
3540  */
3541 int
3542 i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
3543 {
3544         uint32_t old_write_domain, old_read_domains;
3545         int ret;
3546
3547         ret = i915_gem_object_wait_rendering(obj, !write);
3548         if (ret)
3549                 return ret;
3550
3551         if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
3552                 return 0;
3553
3554         i915_gem_object_flush_gtt_write_domain(obj);
3555
3556         old_write_domain = obj->base.write_domain;
3557         old_read_domains = obj->base.read_domains;
3558
3559         /* Flush the CPU cache if it's still invalid. */
3560         if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
3561                 i915_gem_clflush_object(obj, false);
3562
3563                 obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
3564         }
3565
3566         /* It should now be out of any other write domains, and we can update
3567          * the domain values for our changes.
3568          */
3569         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
3570
3571         /* If we're writing through the CPU, then the GPU read domains will
3572          * need to be invalidated at next use.
3573          */
3574         if (write) {
3575                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
3576                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
3577         }
3578
3579         trace_i915_gem_object_change_domain(obj,
3580                                             old_read_domains,
3581                                             old_write_domain);
3582
3583         return 0;
3584 }
3585
3586 /* Throttle our rendering by waiting until the ring has completed our requests
3587  * emitted over 20 msec ago.
3588  *
3589  * Note that if we were to use the current jiffies each time around the loop,
3590  * we wouldn't escape the function with any frames outstanding if the time to
3591  * render a frame was over 20ms.
3592  *
3593  * This should get us reasonable parallelism between CPU and GPU but also
3594  * relatively low latency when blocking on a particular request to finish.
3595  */
3596 static int
3597 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
3598 {
3599         struct drm_i915_private *dev_priv = to_i915(dev);
3600         struct drm_i915_file_private *file_priv = file->driver_priv;
3601         unsigned long recent_enough = jiffies - DRM_I915_THROTTLE_JIFFIES;
3602         struct drm_i915_gem_request *request, *target = NULL;
3603         int ret;
3604
3605         ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
3606         if (ret)
3607                 return ret;
3608
3609         /* ABI: return -EIO if already wedged */
3610         if (i915_terminally_wedged(&dev_priv->gpu_error))
3611                 return -EIO;
3612
3613         spin_lock(&file_priv->mm.lock);
3614         list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
3615                 if (time_after_eq(request->emitted_jiffies, recent_enough))
3616                         break;
3617
3618                 /*
3619                  * Note that the request might not have been submitted yet.
3620                  * In which case emitted_jiffies will be zero.
3621                  */
3622                 if (!request->emitted_jiffies)
3623                         continue;
3624
3625                 target = request;
3626         }
3627         if (target)
3628                 i915_gem_request_get(target);
3629         spin_unlock(&file_priv->mm.lock);
3630
3631         if (target == NULL)
3632                 return 0;
3633
3634         ret = i915_wait_request(target, true, NULL, NULL);
3635         i915_gem_request_put(target);
3636
3637         return ret;
3638 }
3639
3640 static bool
3641 i915_vma_misplaced(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
3642 {
3643         struct drm_i915_gem_object *obj = vma->obj;
3644
3645         if (!drm_mm_node_allocated(&vma->node))
3646                 return false;
3647
3648         if (vma->node.size < size)
3649                 return true;
3650
3651         if (alignment && vma->node.start & (alignment - 1))
3652                 return true;
3653
3654         if (flags & PIN_MAPPABLE && !obj->map_and_fenceable)
3655                 return true;
3656
3657         if (flags & PIN_OFFSET_BIAS &&
3658             vma->node.start < (flags & PIN_OFFSET_MASK))
3659                 return true;
3660
3661         if (flags & PIN_OFFSET_FIXED &&
3662             vma->node.start != (flags & PIN_OFFSET_MASK))
3663                 return true;
3664
3665         return false;
3666 }
3667
3668 void __i915_vma_set_map_and_fenceable(struct i915_vma *vma)
3669 {
3670         struct drm_i915_gem_object *obj = vma->obj;
3671         struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
3672         bool mappable, fenceable;
3673         u32 fence_size, fence_alignment;
3674
3675         fence_size = i915_gem_get_ggtt_size(dev_priv,
3676                                             obj->base.size,
3677                                             i915_gem_object_get_tiling(obj));
3678         fence_alignment = i915_gem_get_ggtt_alignment(dev_priv,
3679                                                       obj->base.size,
3680                                                       i915_gem_object_get_tiling(obj),
3681                                                       true);
3682
3683         fenceable = (vma->node.size == fence_size &&
3684                      (vma->node.start & (fence_alignment - 1)) == 0);
3685
3686         mappable = (vma->node.start + fence_size <=
3687                     dev_priv->ggtt.mappable_end);
3688
3689         obj->map_and_fenceable = mappable && fenceable;
3690 }
3691
3692 int __i915_vma_do_pin(struct i915_vma *vma,
3693                       u64 size, u64 alignment, u64 flags)
3694 {
3695         unsigned int bound = vma->flags;
3696         int ret;
3697
3698         GEM_BUG_ON((flags & (PIN_GLOBAL | PIN_USER)) == 0);
3699         GEM_BUG_ON((flags & PIN_GLOBAL) && !i915_vma_is_ggtt(vma));
3700
3701         if (WARN_ON(bound & I915_VMA_PIN_OVERFLOW)) {
3702                 ret = -EBUSY;
3703                 goto err;
3704         }
3705
3706         if ((bound & I915_VMA_BIND_MASK) == 0) {
3707                 ret = i915_vma_insert(vma, size, alignment, flags);
3708                 if (ret)
3709                         goto err;
3710         }
3711
3712         ret = i915_vma_bind(vma, vma->obj->cache_level, flags);
3713         if (ret)
3714                 goto err;
3715
3716         if ((bound ^ vma->flags) & I915_VMA_GLOBAL_BIND)
3717                 __i915_vma_set_map_and_fenceable(vma);
3718
3719         GEM_BUG_ON(i915_vma_misplaced(vma, size, alignment, flags));
3720         return 0;
3721
3722 err:
3723         __i915_vma_unpin(vma);
3724         return ret;
3725 }
3726
3727 int
3728 i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
3729                          const struct i915_ggtt_view *view,
3730                          u64 size,
3731                          u64 alignment,
3732                          u64 flags)
3733 {
3734         struct i915_vma *vma;
3735         int ret;
3736
3737         if (!view)
3738                 view = &i915_ggtt_view_normal;
3739
3740         vma = i915_gem_obj_lookup_or_create_ggtt_vma(obj, view);
3741         if (IS_ERR(vma))
3742                 return PTR_ERR(vma);
3743
3744         if (i915_vma_misplaced(vma, size, alignment, flags)) {
3745                 if (flags & PIN_NONBLOCK &&
3746                     (i915_vma_is_pinned(vma) || i915_vma_is_active(vma)))
3747                         return -ENOSPC;
3748
3749                 WARN(i915_vma_is_pinned(vma),
3750                      "bo is already pinned in ggtt with incorrect alignment:"
3751                      " offset=%08x %08x, req.alignment=%llx, req.map_and_fenceable=%d,"
3752                      " obj->map_and_fenceable=%d\n",
3753                      upper_32_bits(vma->node.start),
3754                      lower_32_bits(vma->node.start),
3755                      alignment,
3756                      !!(flags & PIN_MAPPABLE),
3757                      obj->map_and_fenceable);
3758                 ret = i915_vma_unbind(vma);
3759                 if (ret)
3760                         return ret;
3761         }
3762
3763         return i915_vma_pin(vma, size, alignment, flags | PIN_GLOBAL);
3764 }
3765
3766 void
3767 i915_gem_object_ggtt_unpin_view(struct drm_i915_gem_object *obj,
3768                                 const struct i915_ggtt_view *view)
3769 {
3770         i915_vma_unpin(i915_gem_obj_to_ggtt_view(obj, view));
3771 }
3772
3773 static __always_inline unsigned int __busy_read_flag(unsigned int id)
3774 {
3775         /* Note that we could alias engines in the execbuf API, but
3776          * that would be very unwise as it prevents userspace from
3777          * fine control over engine selection. Ahem.
3778          *
3779          * This should be something like EXEC_MAX_ENGINE instead of
3780          * I915_NUM_ENGINES.
3781          */
3782         BUILD_BUG_ON(I915_NUM_ENGINES > 16);
3783         return 0x10000 << id;
3784 }
3785
3786 static __always_inline unsigned int __busy_write_id(unsigned int id)
3787 {
3788         /* The uABI guarantees an active writer is also amongst the read
3789          * engines. This would be true if we accessed the activity tracking
3790          * under the lock, but as we perform the lookup of the object and
3791          * its activity locklessly we can not guarantee that the last_write
3792          * being active implies that we have set the same engine flag from
3793          * last_read - hence we always set both read and write busy for
3794          * last_write.
3795          */
3796         return id | __busy_read_flag(id);
3797 }
3798
3799 static __always_inline unsigned int
3800 __busy_set_if_active(const struct i915_gem_active *active,
3801                      unsigned int (*flag)(unsigned int id))
3802 {
3803         /* For more discussion about the barriers and locking concerns,
3804          * see __i915_gem_active_get_rcu().
3805          */
3806         do {
3807                 struct drm_i915_gem_request *request;
3808                 unsigned int id;
3809
3810                 request = rcu_dereference(active->request);
3811                 if (!request || i915_gem_request_completed(request))
3812                         return 0;
3813
3814                 id = request->engine->exec_id;
3815
3816                 /* Check that the pointer wasn't reassigned and overwritten.
3817                  *
3818                  * In __i915_gem_active_get_rcu(), we enforce ordering between
3819                  * the first rcu pointer dereference (imposing a
3820                  * read-dependency only on access through the pointer) and
3821                  * the second lockless access through the memory barrier
3822                  * following a successful atomic_inc_not_zero(). Here there
3823                  * is no such barrier, and so we must manually insert an
3824                  * explicit read barrier to ensure that the following
3825                  * access occurs after all the loads through the first
3826                  * pointer.
3827                  *
3828                  * It is worth comparing this sequence with
3829                  * raw_write_seqcount_latch() which operates very similarly.
3830                  * The challenge here is the visibility of the other CPU
3831                  * writes to the reallocated request vs the local CPU ordering.
3832                  * Before the other CPU can overwrite the request, it will
3833                  * have updated our active->request and gone through a wmb.
3834                  * During the read here, we want to make sure that the values
3835                  * we see have not been overwritten as we do so - and we do
3836                  * that by serialising the second pointer check with the writes
3837                  * on other other CPUs.
3838                  *
3839                  * The corresponding write barrier is part of
3840                  * rcu_assign_pointer().
3841                  */
3842                 smp_rmb();
3843                 if (request == rcu_access_pointer(active->request))
3844                         return flag(id);
3845         } while (1);
3846 }
3847
3848 static __always_inline unsigned int
3849 busy_check_reader(const struct i915_gem_active *active)
3850 {
3851         return __busy_set_if_active(active, __busy_read_flag);
3852 }
3853
3854 static __always_inline unsigned int
3855 busy_check_writer(const struct i915_gem_active *active)
3856 {
3857         return __busy_set_if_active(active, __busy_write_id);
3858 }
3859
3860 int
3861 i915_gem_busy_ioctl(struct drm_device *dev, void *data,
3862                     struct drm_file *file)
3863 {
3864         struct drm_i915_gem_busy *args = data;
3865         struct drm_i915_gem_object *obj;
3866         unsigned long active;
3867
3868         obj = i915_gem_object_lookup(file, args->handle);
3869         if (!obj)
3870                 return -ENOENT;
3871
3872         args->busy = 0;
3873         active = __I915_BO_ACTIVE(obj);
3874         if (active) {
3875                 int idx;
3876
3877                 /* Yes, the lookups are intentionally racy.
3878                  *
3879                  * First, we cannot simply rely on __I915_BO_ACTIVE. We have
3880                  * to regard the value as stale and as our ABI guarantees
3881                  * forward progress, we confirm the status of each active
3882                  * request with the hardware.
3883                  *
3884                  * Even though we guard the pointer lookup by RCU, that only
3885                  * guarantees that the pointer and its contents remain
3886                  * dereferencable and does *not* mean that the request we
3887                  * have is the same as the one being tracked by the object.
3888                  *
3889                  * Consider that we lookup the request just as it is being
3890                  * retired and freed. We take a local copy of the pointer,
3891                  * but before we add its engine into the busy set, the other
3892                  * thread reallocates it and assigns it to a task on another
3893                  * engine with a fresh and incomplete seqno.
3894                  *
3895                  * So after we lookup the engine's id, we double check that
3896                  * the active request is the same and only then do we add it
3897                  * into the busy set.
3898                  */
3899                 rcu_read_lock();
3900
3901                 for_each_active(active, idx)
3902                         args->busy |= busy_check_reader(&obj->last_read[idx]);
3903
3904                 /* For ABI sanity, we only care that the write engine is in
3905                  * the set of read engines. This should be ensured by the
3906                  * ordering of setting last_read/last_write in
3907                  * i915_vma_move_to_active(), and then in reverse in retire.
3908                  * However, for good measure, we always report the last_write
3909                  * request as a busy read as well as being a busy write.
3910                  *
3911                  * We don't care that the set of active read/write engines
3912                  * may change during construction of the result, as it is
3913                  * equally liable to change before userspace can inspect
3914                  * the result.
3915                  */
3916                 args->busy |= busy_check_writer(&obj->last_write);
3917
3918                 rcu_read_unlock();
3919         }
3920
3921         i915_gem_object_put_unlocked(obj);
3922         return 0;
3923 }
3924
3925 int
3926 i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
3927                         struct drm_file *file_priv)
3928 {
3929         return i915_gem_ring_throttle(dev, file_priv);
3930 }
3931
3932 int
3933 i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
3934                        struct drm_file *file_priv)
3935 {
3936         struct drm_i915_private *dev_priv = to_i915(dev);
3937         struct drm_i915_gem_madvise *args = data;
3938         struct drm_i915_gem_object *obj;
3939         int ret;
3940
3941         switch (args->madv) {
3942         case I915_MADV_DONTNEED:
3943         case I915_MADV_WILLNEED:
3944             break;
3945         default:
3946             return -EINVAL;
3947         }
3948
3949         ret = i915_mutex_lock_interruptible(dev);
3950         if (ret)
3951                 return ret;
3952
3953         obj = i915_gem_object_lookup(file_priv, args->handle);
3954         if (!obj) {
3955                 ret = -ENOENT;
3956                 goto unlock;
3957         }
3958
3959         if (obj->pages &&
3960             i915_gem_object_is_tiled(obj) &&
3961             dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
3962                 if (obj->madv == I915_MADV_WILLNEED)
3963                         i915_gem_object_unpin_pages(obj);
3964                 if (args->madv == I915_MADV_WILLNEED)
3965                         i915_gem_object_pin_pages(obj);
3966         }
3967
3968         if (obj->madv != __I915_MADV_PURGED)
3969                 obj->madv = args->madv;
3970
3971         /* if the object is no longer attached, discard its backing storage */
3972         if (obj->madv == I915_MADV_DONTNEED && obj->pages == NULL)
3973                 i915_gem_object_truncate(obj);
3974
3975         args->retained = obj->madv != __I915_MADV_PURGED;
3976
3977         i915_gem_object_put(obj);
3978 unlock:
3979         mutex_unlock(&dev->struct_mutex);
3980         return ret;
3981 }
3982
3983 void i915_gem_object_init(struct drm_i915_gem_object *obj,
3984                           const struct drm_i915_gem_object_ops *ops)
3985 {
3986         int i;
3987
3988         INIT_LIST_HEAD(&obj->global_list);
3989         for (i = 0; i < I915_NUM_ENGINES; i++)
3990                 init_request_active(&obj->last_read[i],
3991                                     i915_gem_object_retire__read);
3992         init_request_active(&obj->last_write,
3993                             i915_gem_object_retire__write);
3994         init_request_active(&obj->last_fence, NULL);
3995         INIT_LIST_HEAD(&obj->obj_exec_link);
3996         INIT_LIST_HEAD(&obj->vma_list);
3997         INIT_LIST_HEAD(&obj->batch_pool_link);
3998
3999         obj->ops = ops;
4000
4001         obj->fence_reg = I915_FENCE_REG_NONE;
4002         obj->madv = I915_MADV_WILLNEED;
4003
4004         i915_gem_info_add_obj(to_i915(obj->base.dev), obj->base.size);
4005 }
4006
4007 static const struct drm_i915_gem_object_ops i915_gem_object_ops = {
4008         .flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE,
4009         .get_pages = i915_gem_object_get_pages_gtt,
4010         .put_pages = i915_gem_object_put_pages_gtt,
4011 };
4012
4013 struct drm_i915_gem_object *i915_gem_object_create(struct drm_device *dev,
4014                                                   size_t size)
4015 {
4016         struct drm_i915_gem_object *obj;
4017         struct address_space *mapping;
4018         gfp_t mask;
4019         int ret;
4020
4021         obj = i915_gem_object_alloc(dev);
4022         if (obj == NULL)
4023                 return ERR_PTR(-ENOMEM);
4024
4025         ret = drm_gem_object_init(dev, &obj->base, size);
4026         if (ret)
4027                 goto fail;
4028
4029         mask = GFP_HIGHUSER | __GFP_RECLAIMABLE;
4030         if (IS_CRESTLINE(dev) || IS_BROADWATER(dev)) {
4031                 /* 965gm cannot relocate objects above 4GiB. */
4032                 mask &= ~__GFP_HIGHMEM;
4033                 mask |= __GFP_DMA32;
4034         }
4035
4036         mapping = file_inode(obj->base.filp)->i_mapping;
4037         mapping_set_gfp_mask(mapping, mask);
4038
4039         i915_gem_object_init(obj, &i915_gem_object_ops);
4040
4041         obj->base.write_domain = I915_GEM_DOMAIN_CPU;
4042         obj->base.read_domains = I915_GEM_DOMAIN_CPU;
4043
4044         if (HAS_LLC(dev)) {
4045                 /* On some devices, we can have the GPU use the LLC (the CPU
4046                  * cache) for about a 10% performance improvement
4047                  * compared to uncached.  Graphics requests other than
4048                  * display scanout are coherent with the CPU in
4049                  * accessing this cache.  This means in this mode we
4050                  * don't need to clflush on the CPU side, and on the
4051                  * GPU side we only need to flush internal caches to
4052                  * get data visible to the CPU.
4053                  *
4054                  * However, we maintain the display planes as UC, and so
4055                  * need to rebind when first used as such.
4056                  */
4057                 obj->cache_level = I915_CACHE_LLC;
4058         } else
4059                 obj->cache_level = I915_CACHE_NONE;
4060
4061         trace_i915_gem_object_create(obj);
4062
4063         return obj;
4064
4065 fail:
4066         i915_gem_object_free(obj);
4067
4068         return ERR_PTR(ret);
4069 }
4070
4071 static bool discard_backing_storage(struct drm_i915_gem_object *obj)
4072 {
4073         /* If we are the last user of the backing storage (be it shmemfs
4074          * pages or stolen etc), we know that the pages are going to be
4075          * immediately released. In this case, we can then skip copying
4076          * back the contents from the GPU.
4077          */
4078
4079         if (obj->madv != I915_MADV_WILLNEED)
4080                 return false;
4081
4082         if (obj->base.filp == NULL)
4083                 return true;
4084
4085         /* At first glance, this looks racy, but then again so would be
4086          * userspace racing mmap against close. However, the first external
4087          * reference to the filp can only be obtained through the
4088          * i915_gem_mmap_ioctl() which safeguards us against the user
4089          * acquiring such a reference whilst we are in the middle of
4090          * freeing the object.
4091          */
4092         return atomic_long_read(&obj->base.filp->f_count) == 1;
4093 }
4094
4095 void i915_gem_free_object(struct drm_gem_object *gem_obj)
4096 {
4097         struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
4098         struct drm_device *dev = obj->base.dev;
4099         struct drm_i915_private *dev_priv = to_i915(dev);
4100         struct i915_vma *vma, *next;
4101
4102         intel_runtime_pm_get(dev_priv);
4103
4104         trace_i915_gem_object_destroy(obj);
4105
4106         /* All file-owned VMA should have been released by this point through
4107          * i915_gem_close_object(), or earlier by i915_gem_context_close().
4108          * However, the object may also be bound into the global GTT (e.g.
4109          * older GPUs without per-process support, or for direct access through
4110          * the GTT either for the user or for scanout). Those VMA still need to
4111          * unbound now.
4112          */
4113         list_for_each_entry_safe(vma, next, &obj->vma_list, obj_link) {
4114                 GEM_BUG_ON(!i915_vma_is_ggtt(vma));
4115                 GEM_BUG_ON(i915_vma_is_active(vma));
4116                 vma->flags &= ~I915_VMA_PIN_MASK;
4117                 i915_vma_close(vma);
4118         }
4119         GEM_BUG_ON(obj->bind_count);
4120
4121         /* Stolen objects don't hold a ref, but do hold pin count. Fix that up
4122          * before progressing. */
4123         if (obj->stolen)
4124                 i915_gem_object_unpin_pages(obj);
4125
4126         WARN_ON(atomic_read(&obj->frontbuffer_bits));
4127
4128         if (obj->pages && obj->madv == I915_MADV_WILLNEED &&
4129             dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES &&
4130             i915_gem_object_is_tiled(obj))
4131                 i915_gem_object_unpin_pages(obj);
4132
4133         if (WARN_ON(obj->pages_pin_count))
4134                 obj->pages_pin_count = 0;
4135         if (discard_backing_storage(obj))
4136                 obj->madv = I915_MADV_DONTNEED;
4137         i915_gem_object_put_pages(obj);
4138
4139         BUG_ON(obj->pages);
4140
4141         if (obj->base.import_attach)
4142                 drm_prime_gem_destroy(&obj->base, NULL);
4143
4144         if (obj->ops->release)
4145                 obj->ops->release(obj);
4146
4147         drm_gem_object_release(&obj->base);
4148         i915_gem_info_remove_obj(dev_priv, obj->base.size);
4149
4150         kfree(obj->bit_17);
4151         i915_gem_object_free(obj);
4152
4153         intel_runtime_pm_put(dev_priv);
4154 }
4155
4156 struct i915_vma *i915_gem_obj_to_vma(struct drm_i915_gem_object *obj,
4157                                      struct i915_address_space *vm)
4158 {
4159         struct i915_vma *vma;
4160         list_for_each_entry(vma, &obj->vma_list, obj_link) {
4161                 if (vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL &&
4162                     vma->vm == vm)
4163                         return vma;
4164         }
4165         return NULL;
4166 }
4167
4168 struct i915_vma *i915_gem_obj_to_ggtt_view(struct drm_i915_gem_object *obj,
4169                                            const struct i915_ggtt_view *view)
4170 {
4171         struct i915_vma *vma;
4172
4173         GEM_BUG_ON(!view);
4174
4175         list_for_each_entry(vma, &obj->vma_list, obj_link)
4176                 if (i915_vma_is_ggtt(vma) &&
4177                     i915_ggtt_view_equal(&vma->ggtt_view, view))
4178                         return vma;
4179         return NULL;
4180 }
4181
4182 int i915_gem_suspend(struct drm_device *dev)
4183 {
4184         struct drm_i915_private *dev_priv = to_i915(dev);
4185         int ret;
4186
4187         intel_suspend_gt_powersave(dev_priv);
4188
4189         mutex_lock(&dev->struct_mutex);
4190
4191         /* We have to flush all the executing contexts to main memory so
4192          * that they can saved in the hibernation image. To ensure the last
4193          * context image is coherent, we have to switch away from it. That
4194          * leaves the dev_priv->kernel_context still active when
4195          * we actually suspend, and its image in memory may not match the GPU
4196          * state. Fortunately, the kernel_context is disposable and we do
4197          * not rely on its state.
4198          */
4199         ret = i915_gem_switch_to_kernel_context(dev_priv);
4200         if (ret)
4201                 goto err;
4202
4203         ret = i915_gem_wait_for_idle(dev_priv, true);
4204         if (ret)
4205                 goto err;
4206
4207         i915_gem_retire_requests(dev_priv);
4208
4209         i915_gem_context_lost(dev_priv);
4210         mutex_unlock(&dev->struct_mutex);
4211
4212         cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work);
4213         cancel_delayed_work_sync(&dev_priv->gt.retire_work);
4214         flush_delayed_work(&dev_priv->gt.idle_work);
4215
4216         /* Assert that we sucessfully flushed all the work and
4217          * reset the GPU back to its idle, low power state.
4218          */
4219         WARN_ON(dev_priv->gt.awake);
4220
4221         return 0;
4222
4223 err:
4224         mutex_unlock(&dev->struct_mutex);
4225         return ret;
4226 }
4227
4228 void i915_gem_resume(struct drm_device *dev)
4229 {
4230         struct drm_i915_private *dev_priv = to_i915(dev);
4231
4232         mutex_lock(&dev->struct_mutex);
4233         i915_gem_restore_gtt_mappings(dev);
4234
4235         /* As we didn't flush the kernel context before suspend, we cannot
4236          * guarantee that the context image is complete. So let's just reset
4237          * it and start again.
4238          */
4239         if (i915.enable_execlists)
4240                 intel_lr_context_reset(dev_priv, dev_priv->kernel_context);
4241
4242         mutex_unlock(&dev->struct_mutex);
4243 }
4244
4245 void i915_gem_init_swizzling(struct drm_device *dev)
4246 {
4247         struct drm_i915_private *dev_priv = to_i915(dev);
4248
4249         if (INTEL_INFO(dev)->gen < 5 ||
4250             dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
4251                 return;
4252
4253         I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
4254                                  DISP_TILE_SURFACE_SWIZZLING);
4255
4256         if (IS_GEN5(dev))
4257                 return;
4258
4259         I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL);
4260         if (IS_GEN6(dev))
4261                 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
4262         else if (IS_GEN7(dev))
4263                 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
4264         else if (IS_GEN8(dev))
4265                 I915_WRITE(GAMTARBMODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_BDW));
4266         else
4267                 BUG();
4268 }
4269
4270 static void init_unused_ring(struct drm_device *dev, u32 base)
4271 {
4272         struct drm_i915_private *dev_priv = to_i915(dev);
4273
4274         I915_WRITE(RING_CTL(base), 0);
4275         I915_WRITE(RING_HEAD(base), 0);
4276         I915_WRITE(RING_TAIL(base), 0);
4277         I915_WRITE(RING_START(base), 0);
4278 }
4279
4280 static void init_unused_rings(struct drm_device *dev)
4281 {
4282         if (IS_I830(dev)) {
4283                 init_unused_ring(dev, PRB1_BASE);
4284                 init_unused_ring(dev, SRB0_BASE);
4285                 init_unused_ring(dev, SRB1_BASE);
4286                 init_unused_ring(dev, SRB2_BASE);
4287                 init_unused_ring(dev, SRB3_BASE);
4288         } else if (IS_GEN2(dev)) {
4289                 init_unused_ring(dev, SRB0_BASE);
4290                 init_unused_ring(dev, SRB1_BASE);
4291         } else if (IS_GEN3(dev)) {
4292                 init_unused_ring(dev, PRB1_BASE);
4293                 init_unused_ring(dev, PRB2_BASE);
4294         }
4295 }
4296
4297 int
4298 i915_gem_init_hw(struct drm_device *dev)
4299 {
4300         struct drm_i915_private *dev_priv = to_i915(dev);
4301         struct intel_engine_cs *engine;
4302         int ret;
4303
4304         /* Double layer security blanket, see i915_gem_init() */
4305         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
4306
4307         if (HAS_EDRAM(dev) && INTEL_GEN(dev_priv) < 9)
4308                 I915_WRITE(HSW_IDICR, I915_READ(HSW_IDICR) | IDIHASHMSK(0xf));
4309
4310         if (IS_HASWELL(dev))
4311                 I915_WRITE(MI_PREDICATE_RESULT_2, IS_HSW_GT3(dev) ?
4312                            LOWER_SLICE_ENABLED : LOWER_SLICE_DISABLED);
4313
4314         if (HAS_PCH_NOP(dev)) {
4315                 if (IS_IVYBRIDGE(dev)) {
4316                         u32 temp = I915_READ(GEN7_MSG_CTL);
4317                         temp &= ~(WAIT_FOR_PCH_FLR_ACK | WAIT_FOR_PCH_RESET_ACK);
4318                         I915_WRITE(GEN7_MSG_CTL, temp);
4319                 } else if (INTEL_INFO(dev)->gen >= 7) {
4320                         u32 temp = I915_READ(HSW_NDE_RSTWRN_OPT);
4321                         temp &= ~RESET_PCH_HANDSHAKE_ENABLE;
4322                         I915_WRITE(HSW_NDE_RSTWRN_OPT, temp);
4323                 }
4324         }
4325
4326         i915_gem_init_swizzling(dev);
4327
4328         /*
4329          * At least 830 can leave some of the unused rings
4330          * "active" (ie. head != tail) after resume which
4331          * will prevent c3 entry. Makes sure all unused rings
4332          * are totally idle.
4333          */
4334         init_unused_rings(dev);
4335
4336         BUG_ON(!dev_priv->kernel_context);
4337
4338         ret = i915_ppgtt_init_hw(dev);
4339         if (ret) {
4340                 DRM_ERROR("PPGTT enable HW failed %d\n", ret);
4341                 goto out;
4342         }
4343
4344         /* Need to do basic initialisation of all rings first: */
4345         for_each_engine(engine, dev_priv) {
4346                 ret = engine->init_hw(engine);
4347                 if (ret)
4348                         goto out;
4349         }
4350
4351         intel_mocs_init_l3cc_table(dev);
4352
4353         /* We can't enable contexts until all firmware is loaded */
4354         ret = intel_guc_setup(dev);
4355         if (ret)
4356                 goto out;
4357
4358 out:
4359         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
4360         return ret;
4361 }
4362
4363 bool intel_sanitize_semaphores(struct drm_i915_private *dev_priv, int value)
4364 {
4365         if (INTEL_INFO(dev_priv)->gen < 6)
4366                 return false;
4367
4368         /* TODO: make semaphores and Execlists play nicely together */
4369         if (i915.enable_execlists)
4370                 return false;
4371
4372         if (value >= 0)
4373                 return value;
4374
4375 #ifdef CONFIG_INTEL_IOMMU
4376         /* Enable semaphores on SNB when IO remapping is off */
4377         if (INTEL_INFO(dev_priv)->gen == 6 && intel_iommu_gfx_mapped)
4378                 return false;
4379 #endif
4380
4381         return true;
4382 }
4383
4384 int i915_gem_init(struct drm_device *dev)
4385 {
4386         struct drm_i915_private *dev_priv = to_i915(dev);
4387         int ret;
4388
4389         mutex_lock(&dev->struct_mutex);
4390
4391         if (!i915.enable_execlists) {
4392                 dev_priv->gt.cleanup_engine = intel_engine_cleanup;
4393         } else {
4394                 dev_priv->gt.cleanup_engine = intel_logical_ring_cleanup;
4395         }
4396
4397         /* This is just a security blanket to placate dragons.
4398          * On some systems, we very sporadically observe that the first TLBs
4399          * used by the CS may be stale, despite us poking the TLB reset. If
4400          * we hold the forcewake during initialisation these problems
4401          * just magically go away.
4402          */
4403         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
4404
4405         i915_gem_init_userptr(dev_priv);
4406
4407         ret = i915_gem_init_ggtt(dev_priv);
4408         if (ret)
4409                 goto out_unlock;
4410
4411         ret = i915_gem_context_init(dev);
4412         if (ret)
4413                 goto out_unlock;
4414
4415         ret = intel_engines_init(dev);
4416         if (ret)
4417                 goto out_unlock;
4418
4419         ret = i915_gem_init_hw(dev);
4420         if (ret == -EIO) {
4421                 /* Allow engine initialisation to fail by marking the GPU as
4422                  * wedged. But we only want to do this where the GPU is angry,
4423                  * for all other failure, such as an allocation failure, bail.
4424                  */
4425                 DRM_ERROR("Failed to initialize GPU, declaring it wedged\n");
4426                 atomic_or(I915_WEDGED, &dev_priv->gpu_error.reset_counter);
4427                 ret = 0;
4428         }
4429
4430 out_unlock:
4431         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
4432         mutex_unlock(&dev->struct_mutex);
4433
4434         return ret;
4435 }
4436
4437 void
4438 i915_gem_cleanup_engines(struct drm_device *dev)
4439 {
4440         struct drm_i915_private *dev_priv = to_i915(dev);
4441         struct intel_engine_cs *engine;
4442
4443         for_each_engine(engine, dev_priv)
4444                 dev_priv->gt.cleanup_engine(engine);
4445 }
4446
4447 static void
4448 init_engine_lists(struct intel_engine_cs *engine)
4449 {
4450         INIT_LIST_HEAD(&engine->request_list);
4451 }
4452
4453 void
4454 i915_gem_load_init_fences(struct drm_i915_private *dev_priv)
4455 {
4456         struct drm_device *dev = &dev_priv->drm;
4457
4458         if (INTEL_INFO(dev_priv)->gen >= 7 && !IS_VALLEYVIEW(dev_priv) &&
4459             !IS_CHERRYVIEW(dev_priv))
4460                 dev_priv->num_fence_regs = 32;
4461         else if (INTEL_INFO(dev_priv)->gen >= 4 || IS_I945G(dev_priv) ||
4462                  IS_I945GM(dev_priv) || IS_G33(dev_priv))
4463                 dev_priv->num_fence_regs = 16;
4464         else
4465                 dev_priv->num_fence_regs = 8;
4466
4467         if (intel_vgpu_active(dev_priv))
4468                 dev_priv->num_fence_regs =
4469                                 I915_READ(vgtif_reg(avail_rs.fence_num));
4470
4471         /* Initialize fence registers to zero */
4472         i915_gem_restore_fences(dev);
4473
4474         i915_gem_detect_bit_6_swizzle(dev);
4475 }
4476
4477 void
4478 i915_gem_load_init(struct drm_device *dev)
4479 {
4480         struct drm_i915_private *dev_priv = to_i915(dev);
4481         int i;
4482
4483         dev_priv->objects =
4484                 kmem_cache_create("i915_gem_object",
4485                                   sizeof(struct drm_i915_gem_object), 0,
4486                                   SLAB_HWCACHE_ALIGN,
4487                                   NULL);
4488         dev_priv->vmas =
4489                 kmem_cache_create("i915_gem_vma",
4490                                   sizeof(struct i915_vma), 0,
4491                                   SLAB_HWCACHE_ALIGN,
4492                                   NULL);
4493         dev_priv->requests =
4494                 kmem_cache_create("i915_gem_request",
4495                                   sizeof(struct drm_i915_gem_request), 0,
4496                                   SLAB_HWCACHE_ALIGN |
4497                                   SLAB_RECLAIM_ACCOUNT |
4498                                   SLAB_DESTROY_BY_RCU,
4499                                   NULL);
4500
4501         INIT_LIST_HEAD(&dev_priv->context_list);
4502         INIT_LIST_HEAD(&dev_priv->mm.unbound_list);
4503         INIT_LIST_HEAD(&dev_priv->mm.bound_list);
4504         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
4505         for (i = 0; i < I915_NUM_ENGINES; i++)
4506                 init_engine_lists(&dev_priv->engine[i]);
4507         for (i = 0; i < I915_MAX_NUM_FENCES; i++)
4508                 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
4509         INIT_DELAYED_WORK(&dev_priv->gt.retire_work,
4510                           i915_gem_retire_work_handler);
4511         INIT_DELAYED_WORK(&dev_priv->gt.idle_work,
4512                           i915_gem_idle_work_handler);
4513         init_waitqueue_head(&dev_priv->gpu_error.wait_queue);
4514         init_waitqueue_head(&dev_priv->gpu_error.reset_queue);
4515
4516         dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
4517
4518         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
4519
4520         init_waitqueue_head(&dev_priv->pending_flip_queue);
4521
4522         dev_priv->mm.interruptible = true;
4523
4524         spin_lock_init(&dev_priv->fb_tracking.lock);
4525 }
4526
4527 void i915_gem_load_cleanup(struct drm_device *dev)
4528 {
4529         struct drm_i915_private *dev_priv = to_i915(dev);
4530
4531         kmem_cache_destroy(dev_priv->requests);
4532         kmem_cache_destroy(dev_priv->vmas);
4533         kmem_cache_destroy(dev_priv->objects);
4534
4535         /* And ensure that our DESTROY_BY_RCU slabs are truly destroyed */
4536         rcu_barrier();
4537 }
4538
4539 int i915_gem_freeze_late(struct drm_i915_private *dev_priv)
4540 {
4541         struct drm_i915_gem_object *obj;
4542
4543         /* Called just before we write the hibernation image.
4544          *
4545          * We need to update the domain tracking to reflect that the CPU
4546          * will be accessing all the pages to create and restore from the
4547          * hibernation, and so upon restoration those pages will be in the
4548          * CPU domain.
4549          *
4550          * To make sure the hibernation image contains the latest state,
4551          * we update that state just before writing out the image.
4552          */
4553
4554         list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list) {
4555                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
4556                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
4557         }
4558
4559         list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
4560                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
4561                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
4562         }
4563
4564         return 0;
4565 }
4566
4567 void i915_gem_release(struct drm_device *dev, struct drm_file *file)
4568 {
4569         struct drm_i915_file_private *file_priv = file->driver_priv;
4570         struct drm_i915_gem_request *request;
4571
4572         /* Clean up our request list when the client is going away, so that
4573          * later retire_requests won't dereference our soon-to-be-gone
4574          * file_priv.
4575          */
4576         spin_lock(&file_priv->mm.lock);
4577         list_for_each_entry(request, &file_priv->mm.request_list, client_list)
4578                 request->file_priv = NULL;
4579         spin_unlock(&file_priv->mm.lock);
4580
4581         if (!list_empty(&file_priv->rps.link)) {
4582                 spin_lock(&to_i915(dev)->rps.client_lock);
4583                 list_del(&file_priv->rps.link);
4584                 spin_unlock(&to_i915(dev)->rps.client_lock);
4585         }
4586 }
4587
4588 int i915_gem_open(struct drm_device *dev, struct drm_file *file)
4589 {
4590         struct drm_i915_file_private *file_priv;
4591         int ret;
4592
4593         DRM_DEBUG_DRIVER("\n");
4594
4595         file_priv = kzalloc(sizeof(*file_priv), GFP_KERNEL);
4596         if (!file_priv)
4597                 return -ENOMEM;
4598
4599         file->driver_priv = file_priv;
4600         file_priv->dev_priv = to_i915(dev);
4601         file_priv->file = file;
4602         INIT_LIST_HEAD(&file_priv->rps.link);
4603
4604         spin_lock_init(&file_priv->mm.lock);
4605         INIT_LIST_HEAD(&file_priv->mm.request_list);
4606
4607         file_priv->bsd_engine = -1;
4608
4609         ret = i915_gem_context_open(dev, file);
4610         if (ret)
4611                 kfree(file_priv);
4612
4613         return ret;
4614 }
4615
4616 /**
4617  * i915_gem_track_fb - update frontbuffer tracking
4618  * @old: current GEM buffer for the frontbuffer slots
4619  * @new: new GEM buffer for the frontbuffer slots
4620  * @frontbuffer_bits: bitmask of frontbuffer slots
4621  *
4622  * This updates the frontbuffer tracking bits @frontbuffer_bits by clearing them
4623  * from @old and setting them in @new. Both @old and @new can be NULL.
4624  */
4625 void i915_gem_track_fb(struct drm_i915_gem_object *old,
4626                        struct drm_i915_gem_object *new,
4627                        unsigned frontbuffer_bits)
4628 {
4629         /* Control of individual bits within the mask are guarded by
4630          * the owning plane->mutex, i.e. we can never see concurrent
4631          * manipulation of individual bits. But since the bitfield as a whole
4632          * is updated using RMW, we need to use atomics in order to update
4633          * the bits.
4634          */
4635         BUILD_BUG_ON(INTEL_FRONTBUFFER_BITS_PER_PIPE * I915_MAX_PIPES >
4636                      sizeof(atomic_t) * BITS_PER_BYTE);
4637
4638         if (old) {
4639                 WARN_ON(!(atomic_read(&old->frontbuffer_bits) & frontbuffer_bits));
4640                 atomic_andnot(frontbuffer_bits, &old->frontbuffer_bits);
4641         }
4642
4643         if (new) {
4644                 WARN_ON(atomic_read(&new->frontbuffer_bits) & frontbuffer_bits);
4645                 atomic_or(frontbuffer_bits, &new->frontbuffer_bits);
4646         }
4647 }
4648
4649 /* All the new VM stuff */
4650 u64 i915_gem_obj_offset(struct drm_i915_gem_object *o,
4651                         struct i915_address_space *vm)
4652 {
4653         struct drm_i915_private *dev_priv = to_i915(o->base.dev);
4654         struct i915_vma *vma;
4655
4656         WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base);
4657
4658         list_for_each_entry(vma, &o->vma_list, obj_link) {
4659                 if (i915_vma_is_ggtt(vma) &&
4660                     vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL)
4661                         continue;
4662                 if (vma->vm == vm)
4663                         return vma->node.start;
4664         }
4665
4666         WARN(1, "%s vma for this object not found.\n",
4667              i915_is_ggtt(vm) ? "global" : "ppgtt");
4668         return -1;
4669 }
4670
4671 u64 i915_gem_obj_ggtt_offset_view(struct drm_i915_gem_object *o,
4672                                   const struct i915_ggtt_view *view)
4673 {
4674         struct i915_vma *vma;
4675
4676         list_for_each_entry(vma, &o->vma_list, obj_link)
4677                 if (i915_vma_is_ggtt(vma) &&
4678                     i915_ggtt_view_equal(&vma->ggtt_view, view))
4679                         return vma->node.start;
4680
4681         WARN(1, "global vma for this object not found. (view=%u)\n", view->type);
4682         return -1;
4683 }
4684
4685 bool i915_gem_obj_bound(struct drm_i915_gem_object *o,
4686                         struct i915_address_space *vm)
4687 {
4688         struct i915_vma *vma;
4689
4690         list_for_each_entry(vma, &o->vma_list, obj_link) {
4691                 if (i915_vma_is_ggtt(vma) &&
4692                     vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL)
4693                         continue;
4694                 if (vma->vm == vm && drm_mm_node_allocated(&vma->node))
4695                         return true;
4696         }
4697
4698         return false;
4699 }
4700
4701 bool i915_gem_obj_ggtt_bound_view(struct drm_i915_gem_object *o,
4702                                   const struct i915_ggtt_view *view)
4703 {
4704         struct i915_vma *vma;
4705
4706         list_for_each_entry(vma, &o->vma_list, obj_link)
4707                 if (i915_vma_is_ggtt(vma) &&
4708                     i915_ggtt_view_equal(&vma->ggtt_view, view) &&
4709                     drm_mm_node_allocated(&vma->node))
4710                         return true;
4711
4712         return false;
4713 }
4714
4715 unsigned long i915_gem_obj_ggtt_size(struct drm_i915_gem_object *o)
4716 {
4717         struct i915_vma *vma;
4718
4719         GEM_BUG_ON(list_empty(&o->vma_list));
4720
4721         list_for_each_entry(vma, &o->vma_list, obj_link) {
4722                 if (i915_vma_is_ggtt(vma) &&
4723                     vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL)
4724                         return vma->node.size;
4725         }
4726
4727         return 0;
4728 }
4729
4730 bool i915_gem_obj_is_pinned(struct drm_i915_gem_object *obj)
4731 {
4732         struct i915_vma *vma;
4733         list_for_each_entry(vma, &obj->vma_list, obj_link)
4734                 if (i915_vma_is_pinned(vma))
4735                         return true;
4736
4737         return false;
4738 }
4739
4740 /* Like i915_gem_object_get_page(), but mark the returned page dirty */
4741 struct page *
4742 i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj, int n)
4743 {
4744         struct page *page;
4745
4746         /* Only default objects have per-page dirty tracking */
4747         if (WARN_ON(!i915_gem_object_has_struct_page(obj)))
4748                 return NULL;
4749
4750         page = i915_gem_object_get_page(obj, n);
4751         set_page_dirty(page);
4752         return page;
4753 }
4754
4755 /* Allocate a new GEM object and fill it with the supplied data */
4756 struct drm_i915_gem_object *
4757 i915_gem_object_create_from_data(struct drm_device *dev,
4758                                  const void *data, size_t size)
4759 {
4760         struct drm_i915_gem_object *obj;
4761         struct sg_table *sg;
4762         size_t bytes;
4763         int ret;
4764
4765         obj = i915_gem_object_create(dev, round_up(size, PAGE_SIZE));
4766         if (IS_ERR(obj))
4767                 return obj;
4768
4769         ret = i915_gem_object_set_to_cpu_domain(obj, true);
4770         if (ret)
4771                 goto fail;
4772
4773         ret = i915_gem_object_get_pages(obj);
4774         if (ret)
4775                 goto fail;
4776
4777         i915_gem_object_pin_pages(obj);
4778         sg = obj->pages;
4779         bytes = sg_copy_from_buffer(sg->sgl, sg->nents, (void *)data, size);
4780         obj->dirty = 1;         /* Backing store is now out of date */
4781         i915_gem_object_unpin_pages(obj);
4782
4783         if (WARN_ON(bytes != size)) {
4784                 DRM_ERROR("Incomplete copy, wrote %zu of %zu", bytes, size);
4785                 ret = -EFAULT;
4786                 goto fail;
4787         }
4788
4789         return obj;
4790
4791 fail:
4792         i915_gem_object_put(obj);
4793         return ERR_PTR(ret);
4794 }